From bb3083064047abd60e1dbf06b4588e18b1481bfb Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Tue, 24 Sep 2024 22:43:39 -0700 Subject: [PATCH 01/11] update --- BuildConfigGen/MakeOptionsReader.cs | 4 +- BuildConfigGen/Program.cs | 735 +++++++++++++----- BuildConfigGen/Properties/launchSettings.json | 2 +- BuildConfigGen/TaskVersion.cs | 17 + docs/subtreeusage.md | 21 + make-options.json | 191 +++++ make-util.js | 10 +- make.js | 39 +- 8 files changed, 806 insertions(+), 213 deletions(-) create mode 100644 docs/subtreeusage.md diff --git a/BuildConfigGen/MakeOptionsReader.cs b/BuildConfigGen/MakeOptionsReader.cs index 6b083a1580b7..7b3d5839f17a 100644 --- a/BuildConfigGen/MakeOptionsReader.cs +++ b/BuildConfigGen/MakeOptionsReader.cs @@ -93,9 +93,9 @@ public AgentTask(string name) Name = name; } - public string Name; + public readonly string Name; - public HashSet Configs = new HashSet(); + public readonly HashSet Configs = new HashSet(); } } diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index 337c27f27832..bfd05f67a4c5 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; -using System.Runtime.InteropServices.JavaScript; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; @@ -25,11 +24,11 @@ internal class Program private const string buildConfigs = "_buildConfigs"; static readonly JsonSerializerOptions jso = new System.Text.Json.JsonSerializerOptions { WriteIndented = true, Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; - static class Config + internal static class Config { public static readonly string[] ExtensionsToPreprocess = new[] { ".ts", ".json" }; - public record ConfigRecord(string name, string constMappingKey, bool isDefault, bool isNode, string nodePackageVersion, bool isWif, string nodeHandler, string preprocessorVariableName, bool enableBuildConfigOverrides, bool deprecated, bool shouldUpdateTypescript, bool writeNpmrc, string? overriddenDirectoryName = null); + public record ConfigRecord(string name, string constMappingKey, bool isDefault, bool isNode, string nodePackageVersion, bool isWif, string nodeHandler, string preprocessorVariableName, bool enableBuildConfigOverrides, bool deprecated, bool shouldUpdateTypescript, bool writeNpmrc, string? overriddenDirectoryName = null, bool shouldUpdateLocalPkgs = false, bool useGlobalVersion = false); public static readonly ConfigRecord Default = new ConfigRecord(name: nameof(Default), constMappingKey: "Default", isDefault: true, isNode: false, nodePackageVersion: "", isWif: false, nodeHandler: "", preprocessorVariableName: "DEFAULT", enableBuildConfigOverrides: false, deprecated: false, shouldUpdateTypescript: false, writeNpmrc: false); public static readonly ConfigRecord Node16 = new ConfigRecord(name: nameof(Node16), constMappingKey: "Node16-219", isDefault: false, isNode: true, nodePackageVersion: "^16.11.39", isWif: false, nodeHandler: "Node16", preprocessorVariableName: "NODE16", enableBuildConfigOverrides: true, deprecated: true, shouldUpdateTypescript: false, writeNpmrc: false); @@ -52,7 +51,8 @@ public record ConfigRecord(string name, string constMappingKey, bool isDefault, public static readonly ConfigRecord Node20_229_14 = new ConfigRecord(name: nameof(Node20_229_14), constMappingKey: "Node20_229_14", isDefault: false, isNode: true, nodePackageVersion: "^20.3.1", isWif: false, nodeHandler: "Node20_1", preprocessorVariableName: "NODE20", enableBuildConfigOverrides: true, deprecated: false, shouldUpdateTypescript: true, overriddenDirectoryName: "Node20", writeNpmrc: true); public static readonly ConfigRecord WorkloadIdentityFederation = new ConfigRecord(name: nameof(WorkloadIdentityFederation), constMappingKey: "WorkloadIdentityFederation", isDefault: false, isNode: true, nodePackageVersion: "^16.11.39", isWif: true, nodeHandler: "Node16", preprocessorVariableName: "WORKLOADIDENTITYFEDERATION", enableBuildConfigOverrides: true, deprecated: false, shouldUpdateTypescript: false, writeNpmrc: false); public static readonly ConfigRecord wif_242 = new ConfigRecord(name: nameof(wif_242), constMappingKey: "wif_242", isDefault: false, isNode: true, nodePackageVersion: "^20.3.1", isWif: true, nodeHandler: "Node20_1", preprocessorVariableName: "WIF", enableBuildConfigOverrides: true, deprecated: false, shouldUpdateTypescript: true, overriddenDirectoryName: "Wif", writeNpmrc: true); - public static ConfigRecord[] Configs = { Default, Node16, Node16_225, Node20, Node20_228, Node20_229_1, Node20_229_2, Node20_229_3, Node20_229_4, Node20_229_5, Node20_229_6, Node20_229_7, Node20_229_8, Node20_229_9, Node20_229_10, Node20_229_11, Node20_229_12, Node20_229_13, Node20_229_14, WorkloadIdentityFederation, wif_242 }; + public static readonly ConfigRecord LocalPackages = new ConfigRecord(name: nameof(LocalPackages), constMappingKey: "LocalPackages", isDefault: false, isNode: true, nodePackageVersion: "^20.3.1", isWif: false, nodeHandler: "Node20_1", preprocessorVariableName: "NODE20", enableBuildConfigOverrides: true, deprecated: false, shouldUpdateTypescript: true, overriddenDirectoryName: "LocalPackages", writeNpmrc: true, shouldUpdateLocalPkgs: true, useGlobalVersion: true); + public static ConfigRecord[] Configs = { Default, Node16, Node16_225, Node20, Node20_228, Node20_229_1, Node20_229_2, Node20_229_3, Node20_229_4, Node20_229_5, Node20_229_6, Node20_229_7, Node20_229_8, Node20_229_9, Node20_229_10, Node20_229_11, Node20_229_12, Node20_229_13, Node20_229_14, WorkloadIdentityFederation, wif_242, LocalPackages }; } static List notSyncronizedDependencies = []; @@ -67,15 +67,21 @@ public record ConfigRecord(string name, string constMappingKey, bool isDefault, /// /// /// When set to the local pipeline agent directory, this tool will produce tasks in debug mode with the corresponding visual studio launch configurations that can be used to attach to built tasks running on this agent - static void Main(string? task = null, string? configs = null, int? currentSprint = null, bool writeUpdates = false, bool allTasks = false, bool getTaskVersionTable = false, string? debugAgentDir = null) + /// Include LocalPackagesBuildConfig + static void Main(string? task = null, string? configs = null, int? currentSprint = null, bool writeUpdates = false, bool allTasks = false, bool getTaskVersionTable = false, string? debugAgentDir = null, bool includeLocalPackagesBuildConfig = false) { try { - MainInner(task, configs, currentSprint, writeUpdates, allTasks, getTaskVersionTable, debugAgentDir); + ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(!writeUpdates); + MainInner(task, configs, currentSprint, writeUpdates, allTasks, getTaskVersionTable, debugAgentDir, includeLocalPackagesBuildConfig); } catch (Exception e2) { // format exceptions nicer than the default formatting. This prevents a long callstack from DragonFruit and puts the exception on the bottom so it's easier to find. + // error handling strategy: + // 1. design: anything goes wrong, try to detect and crash as early as possible to preserve the callstack to make debugging easier. + // 2. we allow all exceptions to fall though. Non-zero exit code will be surfaced + // 3. Ideally default windows exception will occur and errors reported to WER/watson. I'm not sure this is happening, perhaps DragonFruit is handling the exception var restore = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Red; @@ -89,7 +95,7 @@ static void Main(string? task = null, string? configs = null, int? currentSprint } } - private static void MainInner(string? task, string? configs, int? currentSprint, bool writeUpdates, bool allTasks, bool getTaskVersionTable, string? debugAgentDir) + private static void MainInner(string? task, string? configs, int? currentSprintNullable, bool writeUpdates, bool allTasks, bool getTaskVersionTable, string? debugAgentDir, bool includeLocalPackagesBuildConfig) { if (allTasks) { @@ -103,6 +109,12 @@ private static void MainInner(string? task, string? configs, int? currentSprint, string currentDir = Environment.CurrentDirectory; string gitRootPath = GitUtil.GetGitRootPath(currentDir); + + string versionMapFile = Path.Combine(gitRootPath, "_generated", @$"{task}.versionmap.txt"); + string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt"); + + TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath); + if (getTaskVersionTable) { var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); @@ -111,7 +123,7 @@ private static void MainInner(string? task, string? configs, int? currentSprint, foreach (var t in tasks.Values) { - GetVersions(t.Name, string.Join('|', t.Configs), out var r); + GetVersions(t.Name, string.Join('|', t.Configs), out var r, globalVersion); foreach (var z in r) { @@ -126,20 +138,60 @@ private static void MainInner(string? task, string? configs, int? currentSprint, ? new NoDebugConfigGenerator() : new VsCodeLaunchConfigGenerator(gitRootPath, debugAgentDir); + int maxPatchForCurrentSprint = 0; + + int currentSprint; + if (currentSprintNullable.HasValue) + { + currentSprint = currentSprintNullable.Value; + } + else + { + currentSprint = GetCurrentSprint(); + } + + Dictionary taskVersionInfo = []; + if (allTasks) { var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); foreach (var t in tasks.Values) { - MainUpdateTask(t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen); + MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: false); + } + + foreach (var t in tasks.Values) + { + MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, ref globalVersion); + } + + if (includeLocalPackagesBuildConfig) + { + ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); + + foreach (var t in tasks.Values) + { + MainUpdateTask(taskVersionInfo[t.Name], t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); + } } } else { - // error handling strategy: - // 1. design: anything goes wrong, try to detect and crash as early as possible to preserve the callstack to make debugging easier. - // 2. we allow all exceptions to fall though. Non-zero exit code will be surfaced - // 3. Ideally default windows exception will occur and errors reported to WER/watson. I'm not sure this is happening, perhaps DragonFruit is handling the exception + foreach (var t in task!.Split(',', '|')) + { + MainInnerUpdateTaskVersionsInner(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: false); + } + + if (includeLocalPackagesBuildConfig) + { + foreach (var t in task!.Split(',', '|')) + { + MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, ref globalVersion); + } + + ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); + } + foreach (var t in task!.Split(',', '|')) { // If config weren't passed, constract specific config for each task, otherwise we might have problems if first task has 2 configs, but the other 1,0 or diffrenet ones @@ -150,7 +202,7 @@ private static void MainInner(string? task, string? configs, int? currentSprint, { var taskMakeOptions = tasks[t]; var taskConfigs = string.Join('|', taskMakeOptions.Configs); - MainUpdateTask(t, taskConfigs!, writeUpdates, currentSprint, debugConfGen); + MainUpdateTask(taskVersionInfo[t], t, taskConfigs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); } else { @@ -160,11 +212,7 @@ private static void MainInner(string? task, string? configs, int? currentSprint, else { // If configs was passed as arguments, just execute it - MainUpdateTask(t, configs!, writeUpdates, currentSprint, debugConfGen); - } - - if (configs != null) - { + MainUpdateTask(taskVersionInfo[t], t, configs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); } } } @@ -178,6 +226,130 @@ private static void MainInner(string? task, string? configs, int? currentSprint, } } + private static void MainInnerUpdateTaskVersions(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion) + { + // logic -- if includeLocalPackagesBuildConfig, then we want to create/update includeLocalPackagesBuildConfig + // if !includeLocalPackagesBuildConfig -- + // global version NOT mapped to task mapping files + // however, if it exists, account for it when updating existing tasks + + // populate version information + MainInnerUpdateTaskVersionsInner(taskVersionInfo, task, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: false); + } + + private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, ref TaskVersion? globalVersion) + { + if (globalVersion is null) + { + globalVersion = new TaskVersion(0, currentSprint, maxPatchForCurrentSprint); + } + else + { + if (globalVersion.Minor == currentSprint) + { + globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, Math.Max(maxPatchForCurrentSprint, globalVersion.Patch)); + globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[task].configTaskVersionMapping[Config.Default].Major); + } + else + { + globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, 0); + globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[task].configTaskVersionMapping[Config.Default].Major); + } + } + + // populate global verison information + MainInnerUpdateTaskVersionsInner(taskVersionInfo, task!, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: true); + } + + private static void MainInnerUpdateTaskVersionsInner(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + { + // If config weren't passed, constract specific config for each task, otherwise we might have problems if first task has 2 configs, but the other 1,0 or diffrenet ones + if (configs == null) + { + foreach (var t in task!.Split(',', '|')) + { + + var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); + if (tasks.ContainsKey(t)) + { + var taskMakeOptions = tasks[t]; + + HashSet targetConfigs = GetConfigRecords(tasks[t].Configs, writeUpdates); + + UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal); + } + else + { + Console.WriteLine($"Config was not found for the task: {t}, skipping"); + } + } + } + else + { + HashSet targetConfigs = GetConfigRecords(configs!.Split(',', '|'), writeUpdates); + + UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal); + } + } + + private static void UpdateVersionsForTasks(Dictionary taskVersionInfo, string[] tasks, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + { + string currentDir = Environment.CurrentDirectory; + + string gitRootPath = GitUtil.GetGitRootPath(currentDir); + + foreach (var task in tasks) + { + string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); + if (!Directory.Exists(taskTargetPath)) + { + throw new Exception($"expected {taskTargetPath} to exist!"); + } + + Dictionary configTaskVersionMapping; + HashSet versionsUpdated; + + if (taskVersionInfo.TryGetValue(task, out var mapping)) + { + configTaskVersionMapping = mapping.configTaskVersionMapping; + versionsUpdated = mapping.versionsUpdated; + } + else + { + configTaskVersionMapping = []; + versionsUpdated = []; + + taskVersionInfo.Add(task, new TaskStateStruct(configTaskVersionMapping, versionsUpdated)); + } + + if (forGlobal) + { + if (globalVersion is null) + { + throw new Exception("globalVersion shouldn't be null here"); + } + + UpdateVersionsGlobal(task, taskVersionInfo[task], targetConfigs, globalVersion); + } + else + { + UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, versionMapFile, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); + } + + var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); + if (duplicateVersions.Any()) + { + StringBuilder dupConfigsStr = new StringBuilder(); + foreach (var x in duplicateVersions) + { + dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}"); + } + + throw new Exception(dupConfigsStr.ToString()); + } + } + } + private static void NullOrThrow(T value, string message) { if (value != null) @@ -194,7 +366,7 @@ private static void NotNullOrThrow([NotNull] T value, string message) } } - private static void GetVersions(string task, string configsString, out List<(string task, string config, string version)> versionList) + private static void GetVersions(string task, string configsString, out List<(string task, string config, string version)> versionList, TaskVersion? globalVersion) { versionList = new List<(string task, string config, string version)>(); @@ -228,9 +400,7 @@ private static void GetVersions(string task, string configsString, out List<(str try { - ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(false); - - if (ReadVersionMap(versionMapFile, out var versionMap, out var maxVersionNullable)) + if (ReadVersionMap(versionMapFile, out var versionMap, out var maxVersionNullable, globalVersion)) { foreach (var version in versionMap) { @@ -253,64 +423,6 @@ private static void GetVersions(string task, string configsString, out List<(str } } - private static void MainUpdateTask( - string task, - string configsString, - bool writeUpdates, - int? currentSprint, - IDebugConfigGenerator debugConfigGen) - { - if (string.IsNullOrEmpty(task)) - { - throw new Exception("task expected!"); - } - - if (string.IsNullOrEmpty(configsString)) - { - throw new Exception("configs expected!"); - } - - string[] configs = configsString.Split("|"); - string errorMessage; - - Dictionary configdefs = new(Config.Configs.Where(x => !x.isDefault).Select(x => new KeyValuePair(x.name, x))); - HashSet targetConfigs = new HashSet(); - targetConfigs.Add(Config.Default); - foreach (var config in configs) - { - if (configdefs.TryGetValue(config, out var matchedConfig)) - { - if (matchedConfig.deprecated && writeUpdates) - { - errorMessage = "The config with the name: " + matchedConfig.name + " is deprecated. Writing updates for deprecated configs is not allowed."; - throw new Exception(errorMessage); - } - targetConfigs.Add(matchedConfig); - } - else - { - errorMessage = "Configs specified must be one of: " + string.Join(',', Config.Configs.Where(x => !x.isDefault).Select(x => x.name)); - throw new Exception(errorMessage); - } - } - - try - { - ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(!writeUpdates); - - MainUpdateTaskInner(task, currentSprint, targetConfigs, debugConfigGen); - - ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: false); - } - finally - { - if (ensureUpdateModeVerifier != null) - { - ensureUpdateModeVerifier.CleanupTempFiles(); - } - } - } - private static int GetCurrentSprint() { string url = "https://whatsprintis.it"; @@ -342,143 +454,222 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr } } - private static void MainUpdateTaskInner( + private static void MainUpdateTask( + TaskStateStruct taskVersionState, string task, - int? currentSprint, - HashSet targetConfigs, - IDebugConfigGenerator debugConfigGen) + string configsString, + bool writeUpdates, + int currentSprint, + IDebugConfigGenerator debugConfigGen, + bool includeLocalPackagesBuildConfig, + string versionMapFile, + bool hasGlobalVersion + ) { - if (!currentSprint.HasValue) + if (string.IsNullOrEmpty(task)) { - currentSprint = GetCurrentSprint(); + throw new Exception("task expected!"); } - string currentDir = Environment.CurrentDirectory; - - string gitRootPath = GitUtil.GetGitRootPath(currentDir); - - string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); - if (!Directory.Exists(taskTargetPath)) + if (string.IsNullOrEmpty(configsString)) { - throw new Exception($"expected {taskTargetPath} to exist!"); + throw new Exception("configs expected!"); } - string taskHandler = Path.Combine(taskTargetPath, "task.json"); - JsonNode taskHandlerContents = JsonNode.Parse(ensureUpdateModeVerifier!.FileReadAllText(taskHandler))!; - - if (targetConfigs.Any(x => x.isNode)) - { - // Task may not have nodejs or packages.json (example: AutomatedAnalysisV0) - if (!HasNodeHandler(taskHandlerContents)) - { - Console.WriteLine($"Skipping {task} because task doesn't have node handler does not exist"); - return; - } - } + HashSet targetConfigs = GetConfigRecords(configsString.Split("|"), writeUpdates); - // Create _generated - string generatedFolder = Path.Combine(gitRootPath, "_generated"); - if (!Directory.Exists(generatedFolder)) + try { - ensureUpdateModeVerifier!.DirectoryCreateDirectory(generatedFolder, false); - } + string currentDir = Environment.CurrentDirectory; - string versionMapFile = Path.Combine(gitRootPath, "_generated", @$"{task}.versionmap.txt"); + string gitRootPath = GitUtil.GetGitRootPath(currentDir); - UpdateVersions(gitRootPath, task, taskTargetPath, out var configTaskVersionMapping, targetConfigs: targetConfigs, currentSprint.Value, versionMapFile, out HashSet versionsUpdated); - - var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); - if (duplicateVersions.Any()) - { - StringBuilder dupConfigsStr = new StringBuilder(); - foreach (var x in duplicateVersions) + string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); + if (!Directory.Exists(taskTargetPath)) { - dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}"); + throw new Exception($"expected {taskTargetPath} to exist!"); } - throw new Exception(dupConfigsStr.ToString()); - } + string taskHandler = Path.Combine(taskTargetPath, "task.json"); + JsonNode taskHandlerContents = JsonNode.Parse(ensureUpdateModeVerifier!.FileReadAllText(taskHandler))!; - foreach (var config in targetConfigs) - { - bool versionUpdated = versionsUpdated.Contains(config); + if (targetConfigs.Any(x => x.isNode)) + { + // Task may not have nodejs or packages.json (example: AutomatedAnalysisV0) + if (!HasNodeHandler(taskHandlerContents)) + { + Console.WriteLine($"Skipping {task} because task doesn't have node handler does not exist"); + return; + } + } - string taskOutput; - if (config.isDefault) + // Create _generated + string generatedFolder = Path.Combine(gitRootPath, "_generated"); + if (!Directory.Exists(generatedFolder)) { - taskOutput = Path.Combine(gitRootPath, "_generated", task); + ensureUpdateModeVerifier!.DirectoryCreateDirectory(generatedFolder, false); } - else + + foreach (var config in targetConfigs) { - string directoryName = config.name; - if (config.overriddenDirectoryName != null) + if (config.useGlobalVersion && !hasGlobalVersion) { - directoryName = config.overriddenDirectoryName; + Console.WriteLine($"MainUpdateTask: Skipping useGlobalVersion config for task b/c GlobalVersion not initialized. (run with --include-use-local-task-lib-and-common-packages. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion}"); } + else + { + bool versionUpdated = taskVersionState.versionsUpdated.Contains(config); + + string taskOutput; + if (config.isDefault) + { + taskOutput = Path.Combine(gitRootPath, "_generated", task); + } + else + { + string directoryName = config.name; + if (config.overriddenDirectoryName != null) + { + directoryName = config.overriddenDirectoryName; + } + + taskOutput = Path.Combine(gitRootPath, "_generated", @$"{task}_{directoryName}"); + } + + if (config.enableBuildConfigOverrides) + { + EnsureBuildConfigFileOverrides(config, taskTargetPath); + } + + var taskConfigPath = Path.Combine(taskOutput, "task.json"); + var taskConfigExists = File.Exists(taskConfigPath); + + // only update task output if a new version was added, the config exists, the task contains preprocessor instructions, or the config targets Node (not Default) + // Note: CheckTaskInputContainsPreprocessorInstructions is expensive, so only call if needed + if (versionUpdated || taskConfigExists || HasTaskInputContainsPreprocessorInstructions(taskTargetPath, config) || config.isNode) + { + CopyConfig(taskTargetPath, taskOutput, skipPathName: buildConfigs, skipFileName: null, removeExtraFiles: true, throwIfNotUpdatingFileForApplyingOverridesAndPreProcessor: false, config: config, allowPreprocessorDirectives: true); + + if (config.enableBuildConfigOverrides) + { + CopyConfigOverrides(taskTargetPath, taskOutput, config); + } + + // if some files aren't present in destination, stop as following code assumes they're present and we'll just get a FileNotFoundException + // don't check content as preprocessor hasn't run + ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: true); + + HandlePreprocessingInTarget(taskOutput, config, validateAndWriteChanges: true, out _); - taskOutput = Path.Combine(gitRootPath, "_generated", @$"{task}_{directoryName}"); + WriteWIFInputTaskJson(taskOutput, config, "task.json", isLoc: false); + WriteWIFInputTaskJson(taskOutput, config, "task.loc.json", isLoc: true); + WriteTaskJson(taskOutput, taskVersionState.configTaskVersionMapping, config, "task.json"); + WriteTaskJson(taskOutput, taskVersionState.configTaskVersionMapping, config, "task.loc.json"); + } + WriteInputTaskJson(taskTargetPath, taskVersionState.configTaskVersionMapping, "task.json"); + WriteInputTaskJson(taskTargetPath, taskVersionState.configTaskVersionMapping, "task.loc.json"); + + if (config.isNode) + { + GetBuildConfigFileOverridePaths(config, taskTargetPath, out string configTaskPath, out string readmePath); + + string buildConfigPackageJsonPath = Path.Combine(taskTargetPath, buildConfigs, configTaskPath, "package.json"); + + if (File.Exists(buildConfigPackageJsonPath)) + { + EnsureDependencyVersionsAreSyncronized( + task, + Path.Combine(taskTargetPath, "package.json"), + buildConfigPackageJsonPath); + + } + + WriteNodePackageJson(taskOutput, config.nodePackageVersion, config.shouldUpdateTypescript, config.shouldUpdateLocalPkgs); + } + + debugConfigGen.WriteTypescriptConfig(taskOutput); + debugConfigGen.AddForTask(taskConfigPath); + } } - if (config.enableBuildConfigOverrides) + // delay updating version map file until after buildconfigs generated + WriteVersionMapFile(versionMapFile, taskVersionState.configTaskVersionMapping, targetConfigs: targetConfigs); + + ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: false); + } + finally + { + if (ensureUpdateModeVerifier != null) { - EnsureBuildConfigFileOverrides(config, taskTargetPath); + ensureUpdateModeVerifier.CleanupTempFiles(); } + } + } - var taskConfigPath = Path.Combine(taskOutput, "task.json"); - var taskConfigExists = File.Exists(taskConfigPath); + private static HashSet GetConfigRecords(IEnumerable configs, bool writeUpdates) + { + string errorMessage; - // only update task output if a new version was added, the config exists, or the task contains preprocessor instructions - // Note: CheckTaskInputContainsPreprocessorInstructions is expensive, so only call if needed - if (versionUpdated || taskConfigExists || HasTaskInputContainsPreprocessorInstructions(taskTargetPath, config)) + Dictionary configdefs = new(Config.Configs.Where(x => !x.isDefault).Select(x => new KeyValuePair(x.name, x))); + HashSet targetConfigs = new HashSet(); + targetConfigs.Add(Config.Default); + foreach (var config in configs) + { + if (configdefs.TryGetValue(config, out var matchedConfig)) { - CopyConfig(taskTargetPath, taskOutput, skipPathName: buildConfigs, skipFileName: null, removeExtraFiles: true, throwIfNotUpdatingFileForApplyingOverridesAndPreProcessor: false, config: config, allowPreprocessorDirectives: true); - - if (config.enableBuildConfigOverrides) + if (matchedConfig.deprecated && writeUpdates) { - CopyConfigOverrides(taskTargetPath, taskOutput, config); + errorMessage = "The config with the name: " + matchedConfig.name + " is deprecated. Writing updates for deprecated configs is not allowed."; + throw new Exception(errorMessage); } - // if some files aren't present in destination, stop as following code assumes they're present and we'll just get a FileNotFoundException - // don't check content as preprocessor hasn't run - ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: true); - - HandlePreprocessingInTarget(taskOutput, config, validateAndWriteChanges: true, out _); + targetConfigs.Add(matchedConfig); - WriteWIFInputTaskJson(taskOutput, config, "task.json", isLoc: false); - WriteWIFInputTaskJson(taskOutput, config, "task.loc.json", isLoc: true); - WriteTaskJson(taskOutput, configTaskVersionMapping, config, "task.json"); - WriteTaskJson(taskOutput, configTaskVersionMapping, config, "task.loc.json"); } - WriteInputTaskJson(taskTargetPath, configTaskVersionMapping, "task.json"); - WriteInputTaskJson(taskTargetPath, configTaskVersionMapping, "task.loc.json"); - - if (config.isNode) + else { - GetBuildConfigFileOverridePaths(config, taskTargetPath, out string configTaskPath, out string readmePath); - - EnsureDependencyVersionsAreSyncronized( - task, - Path.Combine(taskTargetPath, "package.json"), - Path.Combine(taskTargetPath, buildConfigs, configTaskPath, "package.json")); - WriteNodePackageJson(taskOutput, config.nodePackageVersion, config.shouldUpdateTypescript); + errorMessage = $"Configs ({config}) specified must be one of: " + string.Join(',', Config.Configs.Where(x => !x.isDefault).Select(x => x.name)); + throw new Exception(errorMessage); } - - debugConfigGen.WriteTypescriptConfig(taskOutput); - debugConfigGen.AddForTask(taskConfigPath); } - // delay updating version map file until after buildconfigs generated - WriteVersionMapFile(versionMapFile, configTaskVersionMapping, targetConfigs: targetConfigs); + return targetConfigs; } + + // originVersion buildConfigVersion private static bool VersionIsGreaterThan(string version1, string version2) { + if (version2.StartsWith("^")) + { + // if buildConfig version starts with ^, it's always up to date + return false; + } + const string versionRE = @"(\d+)\.(\d+)\.(\d+)"; var originMatch = Regex.Match(version1, versionRE); - var generatedMatch = Regex.Match(version2, versionRE); - var originDependencyVersion = Version.Parse($"{originMatch.Groups[1].Value}.{originMatch.Groups[2].Value}.{originMatch.Groups[3].Value}"); - var generatedDependencyVersion = Version.Parse($"{generatedMatch.Groups[1].Value}.{generatedMatch.Groups[2].Value}.{generatedMatch.Groups[3].Value}"); - return originDependencyVersion.CompareTo(generatedDependencyVersion) > 0; + var buildConfigMatch = Regex.Match(version2, versionRE); + + if (originMatch.Success && buildConfigMatch.Success) + { + var originDependencyVersion = Version.Parse($"{originMatch.Groups[1].Value}.{originMatch.Groups[2].Value}.{originMatch.Groups[3].Value}"); + var generatedDependencyVersion = Version.Parse($"{buildConfigMatch.Groups[1].Value}.{buildConfigMatch.Groups[2].Value}.{buildConfigMatch.Groups[3].Value}"); + return originDependencyVersion.CompareTo(generatedDependencyVersion) > 0; + } + else + { + if (!originMatch.Success) + { + Console.WriteLine($"VersionIsGreaterThan: {version1} doesn't look like a version"); + } + + if (!buildConfigMatch.Success) + { + Console.WriteLine($"VersionIsGreaterThan: {version2} doesn't look like a version"); + } + } + + return false; } private static void EnsureDependencyVersionsAreSyncronized( @@ -504,15 +695,23 @@ string generatedPackagePath var buildConfigTaskDependencies = buildConfigTaskPackage["dependencies"]; NotNullOrThrow(buildConfigTaskDependencies, $"BUG: buildConfigs dependencies in {task} is null"); string? buildConfigDependencyVersion = buildConfigTaskDependencies[originDependency.Key]?.ToString(); - - if (buildConfigDependencyVersion is null) { + + if (buildConfigDependencyVersion is null) + { notSyncronizedDependencies.Add($@"Dependency ""{originDependency.Key}"" in {task} is missing in buildConfig's package.json"); continue; } - if (VersionIsGreaterThan(originVersion, buildConfigDependencyVersion)) + if (buildConfigDependencyVersion.StartsWith("file:")) // skip if config package is file reference + { + // do nothing + } + else { - notSyncronizedDependencies.Add($@"Dependency ""{originDependency.Key}"" in {generatedPackagePath} has {buildConfigDependencyVersion} version and should be updated to {originVersion} as in {originPackagePath}"); + if (VersionIsGreaterThan(originVersion, buildConfigDependencyVersion)) + { + notSyncronizedDependencies.Add($@"Dependency ""{originDependency.Key}"" in {generatedPackagePath} has {buildConfigDependencyVersion} version and should be updated to {originVersion} as in {originPackagePath}"); + } } } } @@ -696,7 +895,7 @@ private static void WriteWIFInputTaskJson(string taskPath, Config.ConfigRecord c ensureUpdateModeVerifier!.WriteAllText(outputTaskPath, outputTaskNode.ToJsonString(jso), suppressValidationErrorIfTargetPathDoesntExist: false); } - private static void WriteNodePackageJson(string taskOutputNode, string nodeVersion, bool shouldUpdateTypescript) + private static void WriteNodePackageJson(string taskOutputNode, string nodeVersion, bool shouldUpdateTypescript, bool shouldUpdateTaskLib) { string outputNodePackagePath = Path.Combine(taskOutputNode, "package.json"); JsonNode outputNodePackagePathJsonNode = JsonNode.Parse(ensureUpdateModeVerifier!.FileReadAllText(outputNodePackagePath))!; @@ -708,12 +907,42 @@ private static void WriteNodePackageJson(string taskOutputNode, string nodeVersi outputNodePackagePathJsonNode["devDependencies"]!["typescript"] = "5.1.6"; } + if (shouldUpdateTaskLib) + { + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-task-lib", "file:../../task-lib/node/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-artifacts-common", "file:../../tasks-common/common-npm-packages/artifacts-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azp-tasks-az-blobstorage-provider", "file:../../tasks-common/common-npm-packages/az-blobstorage-provider/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azure-arm-rest", "file:../../tasks-common/common-npm-packages/azure-arm-rest/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azurermdeploycommon", "file:../../tasks-common/common-npm-packages/azurermdeploycommon/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codeanalysis-common", "file:../../tasks-common/common-npm-packages/codeanalysis-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codecoverage-tools", "file:../../tasks-common/common-npm-packages/codecoverage-tools/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-docker-common", "file:../../tasks-common/common-npm-packages/docker-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-ios-signing-common", "file:../../tasks-common/common-npm-packages/ios-signing-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-java-common", "file:../../tasks-common/common-npm-packages/java-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-kubernetes-common", "file:../../tasks-common/common-npm-packages/kubernetes-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-msbuildhelpers", "file:../../tasks-common/common-npm-packages/msbuildhelpers/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-packaging-common", "file:../../tasks-common/common-npm-packages/packaging-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-securefiles-common", "file:../../tasks-common/common-npm-packages/securefiles-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-utility-common", "file:../../tasks-common/common-npm-packages/utility-common/_build"); + UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-webdeployment-common", "file:../../tasks-common/common-npm-packages/webdeployment-common/_build"); + } + // We need to add newline since npm install command always add newline at the end of package.json // https://github.com/npm/npm/issues/18545 string nodePackageContent = outputNodePackagePathJsonNode.ToJsonString(jso) + Environment.NewLine; ensureUpdateModeVerifier!.WriteAllText(outputNodePackagePath, nodePackageContent, suppressValidationErrorIfTargetPathDoesntExist: false); } + private static void UpdateDep(JsonNode outputNodePackagePathJsonNode, string module, string buildPath) + { + var depNode = outputNodePackagePathJsonNode["dependencies"]; + var f = depNode![module]; + if (f != null) + { + outputNodePackagePathJsonNode["dependencies"]![module] = buildPath; + } + } + private static bool HasNodeHandler(JsonNode taskHandlerContents) { var possibleExecutionHandlers = new[] { "prejobexecution", "execution", "postjobexecution" }; @@ -839,8 +1068,7 @@ private static void CopyConfig(string taskTargetPathOrUnderscoreBuildConfigPath, } } - - private static void UpdateVersions(string gitRootPath, string task, string taskTarget, out Dictionary configTaskVersionMapping, HashSet targetConfigs, int currentSprint, string versionMapFile, out HashSet versionsUpdated) + private static void UpdateVersions(string task, string taskTarget, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string versionMapFile, string globalVersionPath, ref int maxPatchForCurrentSprint, TaskVersion? globalVersion) { Dictionary versionMap; TaskVersion maxVersion; @@ -851,7 +1079,7 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT { TaskVersion? defaultVersion = null; - if (ReadVersionMap(versionMapFile, out versionMap, out var maxVersionNullable)) + if (ReadVersionMap(versionMapFile, out versionMap, out var maxVersionNullable, globalVersion)) { maxVersion = maxVersionNullable!; defaultVersion = versionMap[Config.Default.name]; @@ -865,19 +1093,17 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT if (inputVersion <= maxVersion && !defaultVersionMatchesSourceVersion) { - throw new Exception($"inputVersion={inputVersion} version specified in task taskTarget={taskTarget} must not be less or equal to maxversion maxVersion={maxVersion} specified in versionMapFile {versionMapFile}, or must match defaultVersion={defaultVersion} in {versionMapFile}"); + throw new Exception($"inputVersion={inputVersion} version specified in task taskTarget={taskTarget} must not be less or equal to maxversion maxVersion={maxVersion} specified in versionMapFile {versionMapFile} and globalVersionPath={globalVersionPath}, or must match defaultVersion={defaultVersion} in {versionMapFile}"); } } - configTaskVersionMapping = new(); - // copy the mappings. As we go check if any configs not mapped. If so, invalidate. bool allConfigsMappedAndValid = true; foreach (var config in targetConfigs) { if (versionMap.ContainsKey(config.constMappingKey)) { - configTaskVersionMapping.Add(config, versionMap[config.constMappingKey]); + taskState.configTaskVersionMapping.Add(config, versionMap[config.constMappingKey]); } else { @@ -888,7 +1114,7 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT // invalidate if input version is not the default version specified in mapping if (allConfigsMappedAndValid) { - if (inputVersion == configTaskVersionMapping[Config.Default]) + if (inputVersion == taskState.configTaskVersionMapping[Config.Default]) { } @@ -898,8 +1124,6 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT } } - - versionsUpdated = new HashSet(); TaskVersion baseVersion = maxVersion; bool baseVersionIsCurrentSprint = baseVersion.Minor == currentSprint; @@ -918,16 +1142,16 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT if (!allConfigsMappedAndValid) { var old = new Dictionary(); - foreach (var x in configTaskVersionMapping) + foreach (var x in taskState.configTaskVersionMapping) { old.Add(x.Key, x.Value); } - configTaskVersionMapping.Clear(); + taskState.configTaskVersionMapping.Clear(); if (defaultVersionMatchesSourceVersion) { - configTaskVersionMapping.Add(Config.Default, inputVersion); + taskState.configTaskVersionMapping.Add(Config.Default, inputVersion); foreach (var config in targetConfigs) { @@ -935,37 +1159,110 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT { if (old.TryGetValue(config, out var oldVersion)) { - configTaskVersionMapping.Add(config, oldVersion); + taskState.configTaskVersionMapping.Add(config, oldVersion); } } } } else { - configTaskVersionMapping.Add(Config.Default, baseVersion.CloneWithPatch(baseVersion.Patch + offset)); + taskState.configTaskVersionMapping.Add(Config.Default, baseVersion.CloneWithPatch(baseVersion.Patch + offset)); offset++; - versionsUpdated.Add(Config.Default); + taskState.versionsUpdated.Add(Config.Default); } foreach (var config in targetConfigs) { - if (!config.isDefault && !configTaskVersionMapping.ContainsKey(config)) + if (!config.isDefault && !taskState.configTaskVersionMapping.ContainsKey(config)) + { + if (config.useGlobalVersion) + { + // global version is updated unconditionally below + } + else + { + TaskVersion targetVersion; + + do + { + targetVersion = baseVersion.CloneWithPatch(baseVersion.Patch + offset); + offset++; + } + while (taskState.configTaskVersionMapping.Values.Contains(targetVersion)); + + taskState.configTaskVersionMapping.Add(config, targetVersion); + + if (!taskState.versionsUpdated.Contains(config)) + { + taskState.versionsUpdated.Add(config); + } + } + } + } + } + + foreach (var x in taskState.configTaskVersionMapping) + { + if (x.Value.Major == currentSprint) + { + maxPatchForCurrentSprint = Math.Max(x.Value.Patch, maxPatchForCurrentSprint); + } + } + } + + private static void UpdateVersionsGlobal(string task, TaskStateStruct taskState, HashSet targetConfigs, TaskVersion globalVersion) + { + foreach (var config in targetConfigs) + { + if (config.useGlobalVersion) + { + if (taskState.configTaskVersionMapping.ContainsKey(config)) { - TaskVersion targetVersion; - do + + if (taskState.configTaskVersionMapping[config] != globalVersion) { - targetVersion = baseVersion.CloneWithPatch(baseVersion.Patch + offset); - offset++; + taskState.configTaskVersionMapping[config] = globalVersion; + + if (!taskState.versionsUpdated.Contains(config)) + { + taskState.versionsUpdated.Add(config); + } } - while (configTaskVersionMapping.Values.Contains(targetVersion)); + } + else + { + taskState.configTaskVersionMapping.Add(config, globalVersion); - configTaskVersionMapping.Add(config, targetVersion); - versionsUpdated.Add(config); + if (!taskState.versionsUpdated.Contains(config)) + { + taskState.versionsUpdated.Add(config); + } } } } } + private static TaskVersion? GetGlobalVersion(string srcPath, string globalVersionPath) + { + if(!File.Exists(globalVersionPath)) + { + return null; + } + + string globalVersionString = File.ReadAllText(globalVersionPath); + + const string shortVersionRegex = @"^\d+\.\d+$"; + Regex re = new Regex(shortVersionRegex); + + if (!re.IsMatch(globalVersionString)) + { + throw new Exception($"{globalVersionPath} doesn't contain expected content matching {shortVersionRegex}"); + } + + TaskVersion globalVersion = new TaskVersion("0." + globalVersionString); + return globalVersion; + } + private static TaskVersion GetInputVersion(string taskTarget) { Int32 patch; @@ -1013,7 +1310,10 @@ private static void WriteVersionMapFile(string versionMapFile, Dictionary versionMap, [NotNullWhen(returnValue: true)] out TaskVersion? maxVersion) + private static bool ReadVersionMap(string versionMapFile, out Dictionary versionMap, [NotNullWhen(returnValue: true)] out TaskVersion? maxVersion, TaskVersion? globalVersion) { versionMap = new(); maxVersion = null; @@ -1097,6 +1398,17 @@ private static bool ReadVersionMap(string versionMapFile, out Dictionary maxVersion) + { + maxVersion = taskGlobalVersion; + } + } + return true; } @@ -1153,4 +1465,17 @@ private static JsonNode GetNodePath(JsonNode node, string path0, params string[] } } + + internal record struct TaskStateStruct(Dictionary configTaskVersionMapping, HashSet versionsUpdated) + { + public static implicit operator (Dictionary configTaskVersionMapping, HashSet versionsUpdated)(TaskStateStruct value) + { + return (value.configTaskVersionMapping, value.versionsUpdated); + } + + public static implicit operator TaskStateStruct((Dictionary configTaskVersionMapping, HashSet versionsUpdated) value) + { + return new TaskStateStruct(value.configTaskVersionMapping, value.versionsUpdated); + } + } } \ No newline at end of file diff --git a/BuildConfigGen/Properties/launchSettings.json b/BuildConfigGen/Properties/launchSettings.json index 6965b0561664..a46a1aebc53d 100644 --- a/BuildConfigGen/Properties/launchSettings.json +++ b/BuildConfigGen/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "BuildConfigGen": { "commandName": "Project", - "commandLineArgs": " --task PipAuthenticateV1 " + "commandLineArgs": " --task AndroidSigningV2 --write-updates " } } } \ No newline at end of file diff --git a/BuildConfigGen/TaskVersion.cs b/BuildConfigGen/TaskVersion.cs index 7cb9a003aaec..bf34a98e6752 100644 --- a/BuildConfigGen/TaskVersion.cs +++ b/BuildConfigGen/TaskVersion.cs @@ -80,6 +80,11 @@ public TaskVersion CloneWithPatch(int overridePatch) return new TaskVersion(Major, Minor, overridePatch); } + public TaskVersion CloneWithMajor(int major) + { + return new TaskVersion(major, Minor, Patch); + } + public static implicit operator String(TaskVersion version) { return version.ToString(); @@ -96,6 +101,18 @@ public override String ToString() return String.Format(CultureInfo.InvariantCulture, "{0}.{1}.{2}{3}", Major, Minor, Patch, suffix); } + + internal string MinorPatchToString() + { + String suffix = String.Empty; + if (IsTest) + { + throw new NotImplementedException(); + } + + return String.Format(CultureInfo.InvariantCulture, "{1}.{2}{3}", Major, Minor, Patch, suffix); + } + public override int GetHashCode() { return this.ToString().GetHashCode(); diff --git a/docs/subtreeusage.md b/docs/subtreeusage.md new file mode 100644 index 000000000000..eab659abd8d1 --- /dev/null +++ b/docs/subtreeusage.md @@ -0,0 +1,21 @@ +Build examples: + +node make.js build --includeLocalPackagesBuildConfig +node make.js build --includeLocalPackagesBuildConfig --task MavenV4 + +Subtrees were added to pull in modules from other repos + +git clone https://github.com/microsoft/azure-pipelines-tasks + + +# add (one time) +git remote add -f azure-pipelines-task-lib https://github.com/microsoft/azure-pipelines-task-lib +git subtree add --prefix task-lib azure-pipelines-task-lib master --squash + +git remote add -f azure-pipelines-tasks-common-packages https://github.com/microsoft/azure-pipelines-tasks-common-packages +git subtree add --prefix tasks-common azure-pipelines-tasks-common-packages main --squash + +# pull +git subtree pull --prefix task-lib azure-pipelines-task-lib master --squash +git subtree pull --prefix tasks-common azure-pipelines-tasks-common-packages main --squash + diff --git a/make-options.json b/make-options.json index 3403bbbfe0ae..9b1deffb9f4f 100644 --- a/make-options.json +++ b/make-options.json @@ -416,5 +416,196 @@ "NuGetAuthenticateV1", "MavenAuthenticateV0", "TwineAuthenticateV1" + ], + "LocalPackages": [ + "AndroidSigningV2", + "AndroidSigningV3", + "ANTV1", + "Not compiling yet: AppCenterDistributeV1", + "Not compiling yet: AppCenterDistributeV2", + "AppCenterDistributeV3", + "AppCenterTestV1", + "ArchiveFilesV2", + "AzureAppConfigurationPushV10", + "AzureAppConfigurationSnapshotV1", + "AzureAppConfigurationV10", + "AzureAppServiceManageV0", + "AzureAppServiceSettingsV1", + "Not compiling yet: AzureCLIV1", + "Not compiling yet: AzureCLIV2", + "AzureCloudPowerShellDeploymentV1", + "AzureCloudPowerShellDeploymentV2", + "Not compiling yet: AzureContainerAppsV0", + "Not compiling yet: AzureContainerAppsV1", + "AzureFileCopyV1", + "AzureFileCopyV2", + "AzureFileCopyV3", + "AzureFileCopyV4", + "AzureFileCopyV5", + "AzureFileCopyV6", + "AzureFunctionAppContainerV1", + "AzureFunctionAppV1", + "AzureFunctionAppV2", + "AzureFunctionOnKubernetesV0", + "AzureFunctionOnKubernetesV1", + "Not compiling yet: AzureIoTEdgeV2", + "AzureKeyVaultV1", + "AzureKeyVaultV2", + "AzureMysqlDeploymentV1", + "Not compiling yet: AzureNLBManagementV1", + "AzurePowerShellV2", + "AzurePowerShellV3", + "AzurePowerShellV4", + "AzurePowerShellV5", + "Not compiling yet: AzureResourceGroupDeploymentV2", + "Not compiling yet: AzureResourceManagerTemplateDeploymentV3", + "AzureRmWebAppDeploymentV3", + "Not compiling yet: AzureRmWebAppDeploymentV4", + "Not compiling yet: AzureSpringCloudV0", + "AzureStaticWebAppV0", + "AzureTestPlanV0", + "AzureVmssDeploymentV0", + "AzureWebAppContainerV1", + "AzureWebAppV1", + "BashV3", + "CargoAuthenticateV0", + "ChefKnifeV1", + "ChefV1", + "CMakeV1", + "CmdLineV2", + "CocoaPodsV0", + "CondaAuthenticateV0", + "CondaEnvironmentV0", + "CondaEnvironmentV1", + "ContainerBuildV0", + "ContainerStructureTestV0", + "CopyFilesOverSSHV0", + "Not compiling yet: CopyFilesV2", + "CUrlUploaderV2", + "DecryptFileV1", + "DeleteFilesV1", + "DockerComposeV0", + "DockerComposeV1", + "DockerInstallerV0", + "DockerV0", + "DockerV1", + "DockerV2", + "Not compiling yet: DotNetCoreCLIV2", + "DotNetCoreInstallerV0", + "DotNetCoreInstallerV1", + "Not compiling yet: DownloadBuildArtifactsV0", + "DownloadFileshareArtifactsV1", + "DownloadGitHubNpmPackageV1", + "Not compiling yet: DownloadGitHubNugetPackageV1", + "Not compiling yet: DownloadGitHubReleaseV0", + "Not compiling yet: DownloadPackageV0", + "Not compiling yet: DownloadPackageV1", + "DownloadSecureFileV1", + "DuffleInstallerV0", + "ExtractFilesV1", + "Not compiling yet: FileTransformV1", + "Not compiling yet: FileTransformV2", + "FtpUploadV1", + "FtpUploadV2", + "FuncToolsInstallerV0", + "GitHubCommentV0", + "Not compiling yet: GitHubReleaseV0", + "Not compiling yet: GitHubReleaseV1", + "GoToolV0", + "GoV0", + "GradleV2", + "GradleV3", + "GruntV0", + "GulpV0", + "GulpV1", + "HelmDeployV0", + "HelmDeployV1", + "HelmInstallerV0", + "HelmInstallerV1", + "IISWebAppDeployment", + "Not compiling yet: IISWebAppDeploymentOnMachineGroupV0", + "IISWebAppManagementOnMachineGroupV0", + "InstallAppleCertificateV2", + "InstallAppleProvisioningProfileV1", + "InstallSSHKeyV0", + "Not compiling yet: JavaToolInstallerV0", + "Not compiling yet: JenkinsDownloadArtifactsV1", + "JenkinsQueueJobV2", + "KubectlInstallerV0", + "Not compiling yet: KubeloginInstallerV0", + "Not compiling yet: KubernetesManifestV0", + "KubernetesManifestV1", + "Not compiling yet: KubernetesV0", + "KubernetesV1", + "MavenAuthenticateV0", + "MavenV2", + "MavenV3", + "MavenV4", + "MSBuildV1", + "Not compiling yet: MysqlDeploymentOnMachineGroupV1", + "NodeTaskRunnerInstallerV0", + "NodeToolV0", + "NotationV0", + "NpmAuthenticateV0", + "Not compiling yet: NpmV0", + "Not compiling yet: NpmV1", + "Not compiling yet: NuGetAuthenticateV0", + "NuGetAuthenticateV1", + "Not compiling yet: NuGetCommandV2", + "Not compiling yet: NuGetInstallerV0", + "Not compiling yet: NuGetPublisherV0", + "Not compiling yet: NuGetRestoreV1", + "Not compiling yet: NuGetToolInstallerV0", + "Not compiling yet (task): NuGetToolInstallerV1", + "Not compiling yet: NuGetV0", + "Not compiling yet: OpenPolicyAgentInstallerV0", + "Not compiling yet: PackerBuildV0", + "Not compiling yet: PackerBuildV1", + "PipAuthenticateV0", + "PipAuthenticateV1", + "PowerShellOnTargetMachinesV1", + "PowerShellOnTargetMachinesV2", + "PowerShellOnTargetMachinesV3", + "PowerShellV2", + "Not compiling yet: PublishBuildArtifactsV1", + "Not compiling yet: PublishCodeCoverageResultsV1", + "PublishCodeCoverageResultsV2", + "Not compiling yet: PublishPipelineMetadataV0", + "Not compiling yet: PublishSymbolsV2", + "Not compiling yet: PublishTestResultsV1", + "Not compiling yet: PublishTestResultsV2", + "PyPIPublisherV0", + "PythonScriptV0", + "QuickPerfTestV1", + "ReviewAppV0", + "RunJMeterLoadTestV1", + "RunLoadTestV1", + "ServiceFabricComposeDeployV0", + "ServiceFabricDeployV1", + "ServiceFabricPowerShellV1", + "ServiceFabricUpdateManifestsV2", + "ShellScriptV2", + "SqlAzureDacpacDeploymentV1", + "SqlDacpacDeploymentOnMachineGroupV0", + "SqlServerDacpacDeployment", + "SshV0", + "TwineAuthenticateV0", + "TwineAuthenticateV1", + "UniversalPackagesV0", + "UseDotNetV2", + "UseNodeV1", + "UsePythonVersionV0", + "UseRubyVersionV0", + "VSBuildV1", + "Not compiling yet: VsTestPlatformToolInstallerV1", + "VsTestV1", + "VsTestV2", + "VsTestV3", + "WindowsMachineFileCopyV1", + "WindowsMachineFileCopyV2", + "XamarinAndroidV1", + "XamariniOSV2", + "XamarinTestCloudV1", + "XcodeV5" ] } diff --git a/make-util.js b/make-util.js index 738a57c9283e..345f5cecc4c3 100644 --- a/make-util.js +++ b/make-util.js @@ -1691,7 +1691,7 @@ const getTaskNodeVersion = function(buildPath, taskName) { return Array.from(nodes); } - console.warn('Unable to determine execution type from task.json, defaulting to use Node 10'); + console.warn('Unable to determine execution type from task.json, defaulting to use Node 10 taskName=' + taskName); nodes.add(10); return Array.from(nodes); } @@ -1778,8 +1778,9 @@ exports.getBuildConfigGenerator = getBuildConfigGenerator; * @param {Boolean} writeUpdates Write Updates (false to validateOnly) * @param {Number} sprintNumber Sprint number option to pass in the BuildConfigGenerator tool * @param {String} debugAgentDir When set to local agent root directory, the BuildConfigGenerator tool will generate launch configurations for the task(s) + * @param {Boolean} includeLocalPackagesBuildConfig When set to true, generate LocalPackages BuildConfig */ -var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, writeUpdates, sprintNumber, debugAgentDir) { +var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, writeUpdates, sprintNumber, debugAgentDir, includeLocalPackagesBuildConfig) { if (!makeOptions) fail("makeOptions is not defined"); if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number"); @@ -1801,6 +1802,11 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, writeUpdateArg += " --write-updates"; } + if(includeLocalPackagesBuildConfig) + { + writeUpdateArg += " --include-local-packages-build-config"; + } + var debugAgentDirArg = ""; if(debugAgentDir) { debugAgentDirArg += ` --debug-agent-dir ${debugAgentDir}`; diff --git a/make.js b/make.js index d98f77b11d72..978e6ee5047e 100644 --- a/make.js +++ b/make.js @@ -57,6 +57,8 @@ var coverageTasksPath = path.join(buildPath, 'coverage'); var baseConfigToolPath = path.join(__dirname, 'BuildConfigGen'); var genTaskPath = path.join(__dirname, '_generated'); var genTaskCommonPath = path.join(__dirname, '_generated', 'Common'); +var taskLibPath = path.join(__dirname, 'task-lib/node'); +var tasksCommonPath = path.join(__dirname, 'tasks-common'); var CLI = {}; @@ -100,6 +102,18 @@ if (argv.task) { } else { // load the default list taskList = fileToJson(makeOptionsPath).tasks; + + if (argv.skipToTask) + { + var skipToTaskIndex = taskList.indexOf(argv.skipToTask); + + if (skipToTaskIndex==-1) + { + fail('argv.skipToTask (' + argv.skipToTask + ') not found'); + } + + taskList = taskList.slice(skipToTaskIndex); + } } // set the runner options. should either be empty or a comma delimited list of test runners. @@ -204,11 +218,30 @@ CLI.serverBuild = async function(/** @type {{ task: string }} */ argv) { } }); + if (argv.includeLocalPackagesBuildConfig) + { + if (!argv.skipPrebuildSteps) + { + // build task-lib + cd(taskLibPath); + run("node make.js build", /*inheritStreams:*/true); + + + await util.installNodeAsync('20'); + // build task-lib + cd(tasksCommonPath); + run("node make.js --build", /*inheritStreams:*/true); + } + } + // Need to validate generated tasks first - const makeOptions = fileToJson(makeOptionsPath); + if (!argv.skipPrebuildSteps) + { + const makeOptions = fileToJson(makeOptionsPath); - // Verify generated files across tasks are up-to-date - util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir']); + // Verify generated files across tasks are up-to-date + util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir'], argv.includeLocalPackagesBuildConfig); + } const allTasks = getTaskList(taskList); From e62dfed46513a87bb7df4269155b9eae5ff4c5cb Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Tue, 24 Sep 2024 23:02:56 -0700 Subject: [PATCH 02/11] fix --- BuildConfigGen/Program.cs | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index bfd05f67a4c5..ed5d173afbdc 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -110,7 +110,6 @@ private static void MainInner(string? task, string? configs, int? currentSprintN string currentDir = Environment.CurrentDirectory; string gitRootPath = GitUtil.GetGitRootPath(currentDir); - string versionMapFile = Path.Combine(gitRootPath, "_generated", @$"{task}.versionmap.txt"); string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt"); TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath); @@ -157,12 +156,12 @@ private static void MainInner(string? task, string? configs, int? currentSprintN var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); foreach (var t in tasks.Values) { - MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: false); + MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); } foreach (var t in tasks.Values) { - MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, ref globalVersion); + MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); } if (includeLocalPackagesBuildConfig) @@ -171,7 +170,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN foreach (var t in tasks.Values) { - MainUpdateTask(taskVersionInfo[t.Name], t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); + MainUpdateTask(taskVersionInfo[t.Name], t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); } } } @@ -179,14 +178,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN { foreach (var t in task!.Split(',', '|')) { - MainInnerUpdateTaskVersionsInner(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal: false); + MainInnerUpdateTaskVersionsInner(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); } if (includeLocalPackagesBuildConfig) { foreach (var t in task!.Split(',', '|')) { - MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, versionMapFile, globalVersionPath, ref globalVersion); + MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); } ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); @@ -202,7 +201,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN { var taskMakeOptions = tasks[t]; var taskConfigs = string.Join('|', taskMakeOptions.Configs); - MainUpdateTask(taskVersionInfo[t], t, taskConfigs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); + MainUpdateTask(taskVersionInfo[t], t, taskConfigs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); } else { @@ -212,7 +211,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN else { // If configs was passed as arguments, just execute it - MainUpdateTask(taskVersionInfo[t], t, configs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, versionMapFile, hasGlobalVersion: globalVersion is not null); + MainUpdateTask(taskVersionInfo[t], t, configs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); } } } @@ -234,10 +233,10 @@ private static void MainInnerUpdateTaskVersions(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, ref TaskVersion? globalVersion) + private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, ref TaskVersion? globalVersion) { if (globalVersion is null) { @@ -258,10 +257,10 @@ private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + private static void MainInnerUpdateTaskVersionsInner(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) { // If config weren't passed, constract specific config for each task, otherwise we might have problems if first task has 2 configs, but the other 1,0 or diffrenet ones if (configs == null) @@ -276,7 +275,7 @@ private static void MainInnerUpdateTaskVersionsInner(Dictionary targetConfigs = GetConfigRecords(tasks[t].Configs, writeUpdates); - UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal); + UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); } else { @@ -288,11 +287,11 @@ private static void MainInnerUpdateTaskVersionsInner(Dictionary targetConfigs = GetConfigRecords(configs!.Split(',', '|'), writeUpdates); - UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, versionMapFile, globalVersionPath, globalVersion, forGlobal); + UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); } } - private static void UpdateVersionsForTasks(Dictionary taskVersionInfo, string[] tasks, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + private static void UpdateVersionsForTasks(Dictionary taskVersionInfo, string[] tasks, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) { string currentDir = Environment.CurrentDirectory; @@ -333,7 +332,7 @@ private static void UpdateVersionsForTasks(Dictionary t } else { - UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, versionMapFile, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); + UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); } var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); @@ -462,7 +461,6 @@ private static void MainUpdateTask( int currentSprint, IDebugConfigGenerator debugConfigGen, bool includeLocalPackagesBuildConfig, - string versionMapFile, bool hasGlobalVersion ) { @@ -483,6 +481,8 @@ bool hasGlobalVersion string currentDir = Environment.CurrentDirectory; string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string versionMapFile = GetVersionMapFile(task, gitRootPath); + string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); if (!Directory.Exists(taskTargetPath)) @@ -606,6 +606,11 @@ bool hasGlobalVersion } } + private static string GetVersionMapFile(string task, string gitRootPath) + { + return Path.Combine(gitRootPath, "_generated", @$"{task}.versionmap.txt"); + } + private static HashSet GetConfigRecords(IEnumerable configs, bool writeUpdates) { string errorMessage; @@ -1068,7 +1073,7 @@ private static void CopyConfig(string taskTargetPathOrUnderscoreBuildConfigPath, } } - private static void UpdateVersions(string task, string taskTarget, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string versionMapFile, string globalVersionPath, ref int maxPatchForCurrentSprint, TaskVersion? globalVersion) + private static void UpdateVersions(string task, string taskTarget, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string globalVersionPath, ref int maxPatchForCurrentSprint, TaskVersion? globalVersion) { Dictionary versionMap; TaskVersion maxVersion; @@ -1077,6 +1082,10 @@ private static void UpdateVersions(string task, string taskTarget, TaskStateStru bool defaultVersionMatchesSourceVersion; + string currentDir = Environment.CurrentDirectory; + string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string versionMapFile = GetVersionMapFile(gitRootPath, task); + { TaskVersion? defaultVersion = null; if (ReadVersionMap(versionMapFile, out versionMap, out var maxVersionNullable, globalVersion)) From 11dbfe94663dce3b29d01fdf80c2b7978efa5c5a Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Tue, 24 Sep 2024 23:34:59 -0700 Subject: [PATCH 03/11] clean up task / config filtering --- BuildConfigGen/Program.cs | 208 ++++++++++++++------------------------ 1 file changed, 75 insertions(+), 133 deletions(-) diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index ed5d173afbdc..5242c195fdc0 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; @@ -151,92 +152,65 @@ private static void MainInner(string? task, string? configs, int? currentSprintN Dictionary taskVersionInfo = []; - if (allTasks) { - var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); - foreach (var t in tasks.Values) - { - MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); - } + var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath).AsEnumerable(); - foreach (var t in tasks.Values) + if (!allTasks) { - MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Name, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); + var taskList = task!.Split(',', '|'); + + tasks = tasks.Where(s => taskList.Where(tl => string.Equals(tl, s.Key, StringComparison.OrdinalIgnoreCase)).Any()); } - if (includeLocalPackagesBuildConfig) + foreach (var t in tasks) { - ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); + IEnumerable configsList = FilterConfigsFotTask(configs, t); - foreach (var t in tasks.Values) - { - MainUpdateTask(taskVersionInfo[t.Name], t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); - } - } - } - else - { - foreach (var t in task!.Split(',', '|')) - { - MainInnerUpdateTaskVersionsInner(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); + MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Value.Name, configsList, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); } if (includeLocalPackagesBuildConfig) { - foreach (var t in task!.Split(',', '|')) + foreach (var t in tasks) { - MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); + IEnumerable configsList = FilterConfigsFotTask(configs, t); + + MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Value.Name, configsList, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); } ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); } - foreach (var t in task!.Split(',', '|')) + foreach (var t in tasks) { - // If config weren't passed, constract specific config for each task, otherwise we might have problems if first task has 2 configs, but the other 1,0 or diffrenet ones - if (configs == null) - { - var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); - if (tasks.ContainsKey(t)) - { - var taskMakeOptions = tasks[t]; - var taskConfigs = string.Join('|', taskMakeOptions.Configs); - MainUpdateTask(taskVersionInfo[t], t, taskConfigs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); - } - else - { - Console.WriteLine($"Config was not found for the task: {t}, skipping"); - } - } - else - { - // If configs was passed as arguments, just execute it - MainUpdateTask(taskVersionInfo[t], t, configs!, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); - } + IEnumerable configsList = FilterConfigsFotTask(configs, t); + + MainUpdateTask(taskVersionInfo[t.Value.Name], t.Value.Name, configsList, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); } - } - debugConfGen.WriteLaunchConfigurations(); + debugConfGen.WriteLaunchConfigurations(); - if (notSyncronizedDependencies.Count > 0) - { - notSyncronizedDependencies.Insert(0, $"##vso[task.logissue type=error]There are problems with the dependencies in the buildConfig's package.json files. Please fix the following issues:"); - throw new Exception(string.Join("\r\n", notSyncronizedDependencies)); + if (notSyncronizedDependencies.Count > 0) + { + notSyncronizedDependencies.Insert(0, $"##vso[task.logissue type=error]There are problems with the dependencies in the buildConfig's package.json files. Please fix the following issues:"); + throw new Exception(string.Join("\r\n", notSyncronizedDependencies)); + } } } - private static void MainInnerUpdateTaskVersions(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string versionMapFile, string globalVersionPath, TaskVersion? globalVersion) + private static IEnumerable FilterConfigsFotTask(string? configs, KeyValuePair t) { - // logic -- if includeLocalPackagesBuildConfig, then we want to create/update includeLocalPackagesBuildConfig - // if !includeLocalPackagesBuildConfig -- - // global version NOT mapped to task mapping files - // however, if it exists, account for it when updating existing tasks + var configsList = t.Value.Configs.AsEnumerable(); + if (configs != null) + { + var configsList2 = configs!.Split(',', '|'); + configsList = configsList.Where(s => configsList2.Where(tl => string.Equals(tl, s, StringComparison.OrdinalIgnoreCase)).Any()); + } - // populate version information - MainInnerUpdateTaskVersionsInner(taskVersionInfo, task, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); + return configsList; } - private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, ref TaskVersion? globalVersion) + private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, IEnumerable configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, ref TaskVersion? globalVersion) { if (globalVersion is null) { @@ -260,92 +234,65 @@ private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, string? configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + private static void MainInnerUpdateTaskVersionsInner(Dictionary taskVersionInfo, string task, IEnumerable configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) { - // If config weren't passed, constract specific config for each task, otherwise we might have problems if first task has 2 configs, but the other 1,0 or diffrenet ones - if (configs == null) - { - foreach (var t in task!.Split(',', '|')) - { - - var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath); - if (tasks.ContainsKey(t)) - { - var taskMakeOptions = tasks[t]; - - HashSet targetConfigs = GetConfigRecords(tasks[t].Configs, writeUpdates); - - UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); - } - else - { - Console.WriteLine($"Config was not found for the task: {t}, skipping"); - } - } - } - else - { - HashSet targetConfigs = GetConfigRecords(configs!.Split(',', '|'), writeUpdates); + HashSet targetConfigs = GetConfigRecords(configs, writeUpdates); - UpdateVersionsForTasks(taskVersionInfo, task!.Split(',', '|'), currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); - } + UpdateVersionsForTask(taskVersionInfo, task, currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); } - private static void UpdateVersionsForTasks(Dictionary taskVersionInfo, string[] tasks, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) + private static void UpdateVersionsForTask(Dictionary taskVersionInfo, string task, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) { string currentDir = Environment.CurrentDirectory; string gitRootPath = GitUtil.GetGitRootPath(currentDir); - foreach (var task in tasks) + string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); + if (!Directory.Exists(taskTargetPath)) { - string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); - if (!Directory.Exists(taskTargetPath)) - { - throw new Exception($"expected {taskTargetPath} to exist!"); - } - - Dictionary configTaskVersionMapping; - HashSet versionsUpdated; + throw new Exception($"expected {taskTargetPath} to exist!"); + } - if (taskVersionInfo.TryGetValue(task, out var mapping)) - { - configTaskVersionMapping = mapping.configTaskVersionMapping; - versionsUpdated = mapping.versionsUpdated; - } - else - { - configTaskVersionMapping = []; - versionsUpdated = []; + Dictionary configTaskVersionMapping; + HashSet versionsUpdated; - taskVersionInfo.Add(task, new TaskStateStruct(configTaskVersionMapping, versionsUpdated)); - } + if (taskVersionInfo.TryGetValue(task, out var mapping)) + { + configTaskVersionMapping = mapping.configTaskVersionMapping; + versionsUpdated = mapping.versionsUpdated; + } + else + { + configTaskVersionMapping = []; + versionsUpdated = []; - if (forGlobal) - { - if (globalVersion is null) - { - throw new Exception("globalVersion shouldn't be null here"); - } + taskVersionInfo.Add(task, new TaskStateStruct(configTaskVersionMapping, versionsUpdated)); + } - UpdateVersionsGlobal(task, taskVersionInfo[task], targetConfigs, globalVersion); - } - else + if (forGlobal) + { + if (globalVersion is null) { - UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); + throw new Exception("globalVersion shouldn't be null here"); } - var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); - if (duplicateVersions.Any()) - { - StringBuilder dupConfigsStr = new StringBuilder(); - foreach (var x in duplicateVersions) - { - dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}"); - } + UpdateVersionsGlobal(task, taskVersionInfo[task], targetConfigs, globalVersion); + } + else + { + UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); + } - throw new Exception(dupConfigsStr.ToString()); + var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); + if (duplicateVersions.Any()) + { + StringBuilder dupConfigsStr = new StringBuilder(); + foreach (var x in duplicateVersions) + { + dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}"); } + + throw new Exception(dupConfigsStr.ToString()); } } @@ -456,7 +403,7 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr private static void MainUpdateTask( TaskStateStruct taskVersionState, string task, - string configsString, + IEnumerable configs, bool writeUpdates, int currentSprint, IDebugConfigGenerator debugConfigGen, @@ -469,12 +416,7 @@ bool hasGlobalVersion throw new Exception("task expected!"); } - if (string.IsNullOrEmpty(configsString)) - { - throw new Exception("configs expected!"); - } - - HashSet targetConfigs = GetConfigRecords(configsString.Split("|"), writeUpdates); + HashSet targetConfigs = GetConfigRecords(configs, writeUpdates); try { From 40cac5500f9e568aeed4d4ac920e3675268832df Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 08:43:39 -0700 Subject: [PATCH 04/11] fix for --all-tasks when invalid tasks are specifed in make-options under specific configs (e..g when a entry is renamed to skip it) --- BuildConfigGen/MakeOptionsReader.cs | 45 +++++++++++++++++++---------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/BuildConfigGen/MakeOptionsReader.cs b/BuildConfigGen/MakeOptionsReader.cs index 7b3d5839f17a..fb92c2051dee 100644 --- a/BuildConfigGen/MakeOptionsReader.cs +++ b/BuildConfigGen/MakeOptionsReader.cs @@ -16,6 +16,7 @@ internal static Dictionary ReadMakeOptions(string gitRootPath var r = new Utf8JsonReader(File.ReadAllBytes(Path.Combine(gitRootPath, @"make-options.json"))); bool inConfig = false; + bool underTasksNode = false; string configName = ""; while (r.Read()) @@ -32,44 +33,56 @@ internal static Dictionary ReadMakeOptions(string gitRootPath { // skip inConfig = false; + underTasksNode = false; } else if (text == "tasks") { inConfig = false; + underTasksNode = true; } else { - inConfig = true; configName = text!; + underTasksNode = false; } break; } case JsonTokenType.String: { - if (inConfig) + if(underTasksNode && inConfig) { - string? text = r.GetString(); - //Console.WriteLine(r.TokenType + " " + text); - - AgentTask task; - if (agentTasks.TryGetValue(text!, out task!)) - { + throw new Exception("don't expect underTasksNode && inConfig"); + } - } - else + // only add tasks under task node! (if there is a task that only exists under a config, ignore it!) + if (underTasksNode) + { + string? text = r.GetString(); + if(agentTasks.ContainsKey(text!)) { - task = new AgentTask(text!); - agentTasks.Add(text!, task); + throw new Exception($"duplicate task in make-options {text}"); } - if (configName == "") + AgentTask task = new AgentTask(text!); + agentTasks.Add(text!, task); + } + + if (inConfig) + { + string? text = r.GetString(); + + AgentTask? task; + if (agentTasks.TryGetValue(text!, out task)) { - throw new Exception("expected configName to have value"); - } + if (configName == "") + { + throw new Exception("expected configName to have value"); + } - task.Configs.Add(configName); + task.Configs.Add(configName); + } } break; From 9a001506bd162d1ebbf8933962bf0e656c690ca4 Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 08:48:02 -0700 Subject: [PATCH 05/11] fix versions getting bumped unexpectedly --- BuildConfigGen/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index 5242c195fdc0..54dc51b0e383 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -1026,7 +1026,7 @@ private static void UpdateVersions(string task, string taskTarget, TaskStateStru string currentDir = Environment.CurrentDirectory; string gitRootPath = GitUtil.GetGitRootPath(currentDir); - string versionMapFile = GetVersionMapFile(gitRootPath, task); + string versionMapFile = GetVersionMapFile(task, gitRootPath); { TaskVersion? defaultVersion = null; From dcfbb22177999d6459e425bc58c0aaa033529bec Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 11:07:40 -0700 Subject: [PATCH 06/11] Avoid writing version mapping files w/only default version: Do not process tasks that don't have any non-default build configs Do not write version map files for tasks that only have defaultbuild and local packages --- BuildConfigGen/Program.cs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index 54dc51b0e383..c1e7f59d4c79 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -153,7 +153,8 @@ private static void MainInner(string? task, string? configs, int? currentSprintN Dictionary taskVersionInfo = []; { - var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath).AsEnumerable(); + var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath).AsEnumerable() + .Where(c=>c.Value.Configs.Any()); // only tasks with configs if (!allTasks) { @@ -1256,19 +1257,26 @@ private static void WriteInputTaskJson(string taskTarget, Dictionary configTaskVersion, HashSet targetConfigs) { - StringBuilder sb = new StringBuilder(); - using (var sw = new StringWriter(sb)) + if (targetConfigs.Where(c => !c.isDefault && !c.useGlobalVersion).Any()) { - foreach (var config in targetConfigs) + StringBuilder sb = new StringBuilder(); + using (var sw = new StringWriter(sb)) { - if (!config.useGlobalVersion) // do not write globalVersion configs to task-specific + foreach (var config in targetConfigs) { - sw.WriteLine(string.Concat(config.constMappingKey, "|", configTaskVersion[config])); + if (!config.useGlobalVersion) // do not write globalVersion configs to task-specific + { + sw.WriteLine(string.Concat(config.constMappingKey, "|", configTaskVersion[config])); + } } } - } - ensureUpdateModeVerifier!.WriteAllText(versionMapFile, sb.ToString(), suppressValidationErrorIfTargetPathDoesntExist: false); + ensureUpdateModeVerifier!.WriteAllText(versionMapFile, sb.ToString(), suppressValidationErrorIfTargetPathDoesntExist: false); + } + else + { + Console.WriteLine($"Not writing {versionMapFile} because there are no configs for task which are not Default or useGlobalVersion"); + } } private static string GetExecutionPath(JsonNode taskNode, string execution) From d821fe2f46a6dbc2df97761adf5b8be193d5c0bb Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 14:56:39 -0700 Subject: [PATCH 07/11] consolidation/simplification --- BuildConfigGen/Program.cs | 177 +++++++++++++++----------------------- 1 file changed, 71 insertions(+), 106 deletions(-) diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index c1e7f59d4c79..3c821b0d5182 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -112,7 +112,6 @@ private static void MainInner(string? task, string? configs, int? currentSprintN string gitRootPath = GitUtil.GetGitRootPath(currentDir); string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt"); - TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath); if (getTaskVersionTable) @@ -154,7 +153,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN { var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath).AsEnumerable() - .Where(c=>c.Value.Configs.Any()); // only tasks with configs + .Where(c => c.Value.Configs.Any()); // only tasks with configs if (!allTasks) { @@ -165,18 +164,51 @@ private static void MainInner(string? task, string? configs, int? currentSprintN foreach (var t in tasks) { - IEnumerable configsList = FilterConfigsFotTask(configs, t); + taskVersionInfo.Add(t.Value.Name, new TaskStateStruct([], [])); + } - MainInnerUpdateTaskVersionsInner(taskVersionInfo, t.Value.Name, configsList, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: false); + foreach (var t in tasks) + { + IEnumerable configsList = FilterConfigsForTask(configs, t); + HashSet targetConfigs = GetConfigRecords(configsList, writeUpdates); + UpdateVersionsForTask(t.Value.Name, taskVersionInfo[t.Value.Name], targetConfigs, currentSprint, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); + CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping); } if (includeLocalPackagesBuildConfig) { foreach (var t in tasks) { - IEnumerable configsList = FilterConfigsFotTask(configs, t); + IEnumerable configsList = FilterConfigsForTask(configs, t); + + if (globalVersion is null) + { + globalVersion = new TaskVersion(0, currentSprint, maxPatchForCurrentSprint); + } + else + { + if (globalVersion.Minor == currentSprint) + { + globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, Math.Max(maxPatchForCurrentSprint, globalVersion.Patch)); + globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[t.Value.Name].configTaskVersionMapping[Config.Default].Major); + } + else + { + globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, 0); + globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[t.Value.Name].configTaskVersionMapping[Config.Default].Major); + } + } + + // populate global verison information + HashSet targetConfigs = GetConfigRecords(configsList, writeUpdates); - MainInnerUpdateTaskVersionsGlobal(taskVersionInfo, t.Value.Name, configsList, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, ref globalVersion); + if (globalVersion is null) + { + throw new Exception("globalVersion shouldn't be null here"); + } + + UpdateVersionsGlobal(t.Value.Name, taskVersionInfo[t.Value.Name], targetConfigs, globalVersion); + CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping); } ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); @@ -184,7 +216,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN foreach (var t in tasks) { - IEnumerable configsList = FilterConfigsFotTask(configs, t); + IEnumerable configsList = FilterConfigsForTask(configs, t); MainUpdateTask(taskVersionInfo[t.Value.Name], t.Value.Name, configsList, writeUpdates, currentSprint, debugConfGen, includeLocalPackagesBuildConfig, hasGlobalVersion: globalVersion is not null); } @@ -199,7 +231,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN } } - private static IEnumerable FilterConfigsFotTask(string? configs, KeyValuePair t) + private static IEnumerable FilterConfigsForTask(string? configs, KeyValuePair t) { var configsList = t.Value.Configs.AsEnumerable(); if (configs != null) @@ -211,79 +243,8 @@ private static IEnumerable FilterConfigsFotTask(string? configs, KeyValu return configsList; } - private static void MainInnerUpdateTaskVersionsGlobal(Dictionary taskVersionInfo, string task, IEnumerable configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, ref TaskVersion? globalVersion) + private static void CheckForDuplicates(string task, Dictionary configTaskVersionMapping) { - if (globalVersion is null) - { - globalVersion = new TaskVersion(0, currentSprint, maxPatchForCurrentSprint); - } - else - { - if (globalVersion.Minor == currentSprint) - { - globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, Math.Max(maxPatchForCurrentSprint, globalVersion.Patch)); - globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[task].configTaskVersionMapping[Config.Default].Major); - } - else - { - globalVersion = globalVersion.CloneWithMinorAndPatch(currentSprint, 0); - globalVersion = globalVersion.CloneWithMajor(taskVersionInfo[task].configTaskVersionMapping[Config.Default].Major); - } - } - - // populate global verison information - MainInnerUpdateTaskVersionsInner(taskVersionInfo, task!, configs, writeUpdates, includeLocalPackagesBuildConfig, gitRootPath, ref maxPatchForCurrentSprint, currentSprint, globalVersionPath, globalVersion, forGlobal: true); - } - - private static void MainInnerUpdateTaskVersionsInner(Dictionary taskVersionInfo, string task, IEnumerable configs, bool writeUpdates, bool includeLocalPackagesBuildConfig, string gitRootPath, ref int maxPatchForCurrentSprint, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) - { - HashSet targetConfigs = GetConfigRecords(configs, writeUpdates); - - UpdateVersionsForTask(taskVersionInfo, task, currentSprint, targetConfigs, ref maxPatchForCurrentSprint, globalVersionPath, globalVersion, forGlobal); - } - - private static void UpdateVersionsForTask(Dictionary taskVersionInfo, string task, int currentSprint, HashSet targetConfigs, ref int maxPatchForCurrentSprint, string globalVersionPath, TaskVersion? globalVersion, bool forGlobal) - { - string currentDir = Environment.CurrentDirectory; - - string gitRootPath = GitUtil.GetGitRootPath(currentDir); - - string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); - if (!Directory.Exists(taskTargetPath)) - { - throw new Exception($"expected {taskTargetPath} to exist!"); - } - - Dictionary configTaskVersionMapping; - HashSet versionsUpdated; - - if (taskVersionInfo.TryGetValue(task, out var mapping)) - { - configTaskVersionMapping = mapping.configTaskVersionMapping; - versionsUpdated = mapping.versionsUpdated; - } - else - { - configTaskVersionMapping = []; - versionsUpdated = []; - - taskVersionInfo.Add(task, new TaskStateStruct(configTaskVersionMapping, versionsUpdated)); - } - - if (forGlobal) - { - if (globalVersion is null) - { - throw new Exception("globalVersion shouldn't be null here"); - } - - UpdateVersionsGlobal(task, taskVersionInfo[task], targetConfigs, globalVersion); - } - else - { - UpdateVersions(task, taskTargetPath, taskVersionInfo[task], targetConfigs, currentSprint, globalVersionPath, ref maxPatchForCurrentSprint, globalVersion); - } - var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1); if (duplicateVersions.Any()) { @@ -422,11 +383,9 @@ bool hasGlobalVersion try { string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); string versionMapFile = GetVersionMapFile(task, gitRootPath); - string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); if (!Directory.Exists(taskTargetPath)) { @@ -857,22 +816,22 @@ private static void WriteNodePackageJson(string taskOutputNode, string nodeVersi if (shouldUpdateTaskLib) { - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-task-lib", "file:../../task-lib/node/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-artifacts-common", "file:../../tasks-common/common-npm-packages/artifacts-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azp-tasks-az-blobstorage-provider", "file:../../tasks-common/common-npm-packages/az-blobstorage-provider/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azure-arm-rest", "file:../../tasks-common/common-npm-packages/azure-arm-rest/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azurermdeploycommon", "file:../../tasks-common/common-npm-packages/azurermdeploycommon/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codeanalysis-common", "file:../../tasks-common/common-npm-packages/codeanalysis-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codecoverage-tools", "file:../../tasks-common/common-npm-packages/codecoverage-tools/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-docker-common", "file:../../tasks-common/common-npm-packages/docker-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-ios-signing-common", "file:../../tasks-common/common-npm-packages/ios-signing-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-java-common", "file:../../tasks-common/common-npm-packages/java-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-kubernetes-common", "file:../../tasks-common/common-npm-packages/kubernetes-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-msbuildhelpers", "file:../../tasks-common/common-npm-packages/msbuildhelpers/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-packaging-common", "file:../../tasks-common/common-npm-packages/packaging-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-securefiles-common", "file:../../tasks-common/common-npm-packages/securefiles-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-utility-common", "file:../../tasks-common/common-npm-packages/utility-common/_build"); - UpdateDep(outputNodePackagePathJsonNode, "azure-pipelines-tasks-webdeployment-common", "file:../../tasks-common/common-npm-packages/webdeployment-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-task-lib", "file:../../task-lib/node/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-artifacts-common", "file:../../tasks-common/common-npm-packages/artifacts-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azp-tasks-az-blobstorage-provider", "file:../../tasks-common/common-npm-packages/az-blobstorage-provider/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azure-arm-rest", "file:../../tasks-common/common-npm-packages/azure-arm-rest/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-azurermdeploycommon", "file:../../tasks-common/common-npm-packages/azurermdeploycommon/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codeanalysis-common", "file:../../tasks-common/common-npm-packages/codeanalysis-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-codecoverage-tools", "file:../../tasks-common/common-npm-packages/codecoverage-tools/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-docker-common", "file:../../tasks-common/common-npm-packages/docker-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-ios-signing-common", "file:../../tasks-common/common-npm-packages/ios-signing-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-java-common", "file:../../tasks-common/common-npm-packages/java-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-kubernetes-common", "file:../../tasks-common/common-npm-packages/kubernetes-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-msbuildhelpers", "file:../../tasks-common/common-npm-packages/msbuildhelpers/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-packaging-common", "file:../../tasks-common/common-npm-packages/packaging-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-securefiles-common", "file:../../tasks-common/common-npm-packages/securefiles-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-utility-common", "file:../../tasks-common/common-npm-packages/utility-common/_build"); + UpdateDepNode(outputNodePackagePathJsonNode, "azure-pipelines-tasks-webdeployment-common", "file:../../tasks-common/common-npm-packages/webdeployment-common/_build"); } // We need to add newline since npm install command always add newline at the end of package.json @@ -881,7 +840,7 @@ private static void WriteNodePackageJson(string taskOutputNode, string nodeVersi ensureUpdateModeVerifier!.WriteAllText(outputNodePackagePath, nodePackageContent, suppressValidationErrorIfTargetPathDoesntExist: false); } - private static void UpdateDep(JsonNode outputNodePackagePathJsonNode, string module, string buildPath) + private static void UpdateDepNode(JsonNode outputNodePackagePathJsonNode, string module, string buildPath) { var depNode = outputNodePackagePathJsonNode["dependencies"]; var f = depNode![module]; @@ -1016,17 +975,24 @@ private static void CopyConfig(string taskTargetPathOrUnderscoreBuildConfigPath, } } - private static void UpdateVersions(string task, string taskTarget, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string globalVersionPath, ref int maxPatchForCurrentSprint, TaskVersion? globalVersion) + private static void UpdateVersionsForTask(string task, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string globalVersionPath, ref int maxPatchForCurrentSprint, TaskVersion? globalVersion) { + string currentDir = Environment.CurrentDirectory; + string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); + + if (!Directory.Exists(taskTargetPath)) + { + throw new Exception($"expected {taskTargetPath} to exist!"); + } + Dictionary versionMap; TaskVersion maxVersion; - var inputVersion = GetInputVersion(taskTarget); + var inputVersion = GetInputVersion(taskTargetPath); bool defaultVersionMatchesSourceVersion; - string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); string versionMapFile = GetVersionMapFile(task, gitRootPath); { @@ -1045,7 +1011,7 @@ private static void UpdateVersions(string task, string taskTarget, TaskStateStru if (inputVersion <= maxVersion && !defaultVersionMatchesSourceVersion) { - throw new Exception($"inputVersion={inputVersion} version specified in task taskTarget={taskTarget} must not be less or equal to maxversion maxVersion={maxVersion} specified in versionMapFile {versionMapFile} and globalVersionPath={globalVersionPath}, or must match defaultVersion={defaultVersion} in {versionMapFile}"); + throw new Exception($"inputVersion={inputVersion} version specified in task taskTargetPath={taskTargetPath} must not be less or equal to maxversion maxVersion={maxVersion} specified in versionMapFile {versionMapFile} and globalVersionPath={globalVersionPath}, or must match defaultVersion={defaultVersion} in {versionMapFile}"); } } @@ -1170,7 +1136,6 @@ private static void UpdateVersionsGlobal(string task, TaskStateStruct taskState, { if (taskState.configTaskVersionMapping.ContainsKey(config)) { - if (taskState.configTaskVersionMapping[config] != globalVersion) { taskState.configTaskVersionMapping[config] = globalVersion; @@ -1196,7 +1161,7 @@ private static void UpdateVersionsGlobal(string task, TaskStateStruct taskState, private static TaskVersion? GetGlobalVersion(string srcPath, string globalVersionPath) { - if(!File.Exists(globalVersionPath)) + if (!File.Exists(globalVersionPath)) { return null; } From 423d8beeedfe9fc69c78b8dee4f176d7169a1e0d Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 14:56:56 -0700 Subject: [PATCH 08/11] doc update --- docs/localpackages_buildconfig.md | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/localpackages_buildconfig.md diff --git a/docs/localpackages_buildconfig.md b/docs/localpackages_buildconfig.md new file mode 100644 index 000000000000..b7e467ea3edb --- /dev/null +++ b/docs/localpackages_buildconfig.md @@ -0,0 +1,33 @@ +## Usage examples + +node make.js build --includeLocalPackagesBuildConfig +node make.js build --includeLocalPackagesBuildConfig --task MavenV4 + +## What does specifing --includeLocalPackagesBuildConfig do? + +It produces a new LocalPackages config, which updates several package.json references to point to local files. + +A single version is used for all LocalPackages, stored in [gitroot]\globalversion.txt. globalversion.txt is only created if "—includeLocalPackagesBuildConfig" is specified. + +globalversion.txt (if present) is updated to avoid clashes with existing task versions. The global version specified is use for tasks generated under LocalPackages build.config. The version number is [major from task].[sprint].[patch] + +This hybrid approach is transitionary. In the future, all build configs (including the default) will use the global version. + +## sub-tree + +Subtrees were added to pull in modules from other repos + +git clone https://github.com/microsoft/azure-pipelines-tasks + + +# add (one time) +git remote add -f azure-pipelines-task-lib https://github.com/microsoft/azure-pipelines-task-lib +git subtree add --prefix task-lib azure-pipelines-task-lib master --squash + +git remote add -f azure-pipelines-tasks-common-packages https://github.com/microsoft/azure-pipelines-tasks-common-packages +git subtree add --prefix tasks-common azure-pipelines-tasks-common-packages main --squash + +# pull +git subtree pull --prefix task-lib azure-pipelines-task-lib master --squash +git subtree pull --prefix tasks-common azure-pipelines-tasks-common-packages main --squash + From f3bbc62977fbea330c7adddbf7da7c1f3b534d48 Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 15:11:30 -0700 Subject: [PATCH 09/11] bump MavenV2/3 for testing (don't merge!) --- Tasks/MavenV3/task.json | 2 +- Tasks/MavenV3/task.loc.json | 2 +- Tasks/MavenV4/task.json | 2 +- Tasks/MavenV4/task.loc.json | 2 +- _generated/MavenV3.versionmap.txt | 4 ++-- _generated/MavenV3/task.json | 6 +++--- _generated/MavenV3/task.loc.json | 6 +++--- _generated/MavenV3_Node20/task.json | 6 +++--- _generated/MavenV3_Node20/task.loc.json | 6 +++--- _generated/MavenV4.versionmap.txt | 4 ++-- _generated/MavenV4/task.json | 6 +++--- _generated/MavenV4/task.loc.json | 6 +++--- _generated/MavenV4_Node20/task.json | 6 +++--- _generated/MavenV4_Node20/task.loc.json | 6 +++--- 14 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Tasks/MavenV3/task.json b/Tasks/MavenV3/task.json index e967a68e223b..030195055bd2 100644 --- a/Tasks/MavenV3/task.json +++ b/Tasks/MavenV3/task.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ diff --git a/Tasks/MavenV3/task.loc.json b/Tasks/MavenV3/task.loc.json index 3db9c5d2230f..6fc457236f29 100644 --- a/Tasks/MavenV3/task.loc.json +++ b/Tasks/MavenV3/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ diff --git a/Tasks/MavenV4/task.json b/Tasks/MavenV4/task.json index 140fd2aa6220..ecd7f054edb0 100644 --- a/Tasks/MavenV4/task.json +++ b/Tasks/MavenV4/task.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ diff --git a/Tasks/MavenV4/task.loc.json b/Tasks/MavenV4/task.loc.json index 64d09b4339b3..39342f30c67d 100644 --- a/Tasks/MavenV4/task.loc.json +++ b/Tasks/MavenV4/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ diff --git a/_generated/MavenV3.versionmap.txt b/_generated/MavenV3.versionmap.txt index 489e7d778db0..b8c4044e680b 100644 --- a/_generated/MavenV3.versionmap.txt +++ b/_generated/MavenV3.versionmap.txt @@ -1,2 +1,2 @@ -Default|3.246.0 -Node20-225|3.246.1 +Default|3.246.2 +Node20-225|3.246.3 diff --git a/_generated/MavenV3/task.json b/_generated/MavenV3/task.json index a466db123837..9e7394cd8376 100644 --- a/_generated/MavenV3/task.json +++ b/_generated/MavenV3/task.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ @@ -519,7 +519,7 @@ "NoTestResults": "No test result files matching %s were found, so publishing JUnit test results is being skipped." }, "_buildConfigMapping": { - "Default": "3.246.0", - "Node20-225": "3.246.1" + "Default": "3.246.2", + "Node20-225": "3.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV3/task.loc.json b/_generated/MavenV3/task.loc.json index a96ec76829e8..a3bf4d9f34cc 100644 --- a/_generated/MavenV3/task.loc.json +++ b/_generated/MavenV3/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ @@ -519,7 +519,7 @@ "NoTestResults": "ms-resource:loc.messages.NoTestResults" }, "_buildConfigMapping": { - "Default": "3.246.0", - "Node20-225": "3.246.1" + "Default": "3.246.2", + "Node20-225": "3.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV3_Node20/task.json b/_generated/MavenV3_Node20/task.json index 2cb02038e0d7..f9ee200371eb 100644 --- a/_generated/MavenV3_Node20/task.json +++ b/_generated/MavenV3_Node20/task.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 1 + "Patch": 3 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ @@ -523,7 +523,7 @@ "NoTestResults": "No test result files matching %s were found, so publishing JUnit test results is being skipped." }, "_buildConfigMapping": { - "Default": "3.246.0", - "Node20-225": "3.246.1" + "Default": "3.246.2", + "Node20-225": "3.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV3_Node20/task.loc.json b/_generated/MavenV3_Node20/task.loc.json index 1ac0ce9c5ae8..e028a2d55469 100644 --- a/_generated/MavenV3_Node20/task.loc.json +++ b/_generated/MavenV3_Node20/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 3, "Minor": 246, - "Patch": 1 + "Patch": 3 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ @@ -523,7 +523,7 @@ "NoTestResults": "ms-resource:loc.messages.NoTestResults" }, "_buildConfigMapping": { - "Default": "3.246.0", - "Node20-225": "3.246.1" + "Default": "3.246.2", + "Node20-225": "3.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV4.versionmap.txt b/_generated/MavenV4.versionmap.txt index 60bf960e27a0..b22dbf5f0ff9 100644 --- a/_generated/MavenV4.versionmap.txt +++ b/_generated/MavenV4.versionmap.txt @@ -1,2 +1,2 @@ -Default|4.246.0 -Node20-225|4.246.1 +Default|4.246.2 +Node20-225|4.246.3 diff --git a/_generated/MavenV4/task.json b/_generated/MavenV4/task.json index 16fff31ce40d..6a6db13cf2e6 100644 --- a/_generated/MavenV4/task.json +++ b/_generated/MavenV4/task.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ @@ -527,7 +527,7 @@ "NoTestResults": "No test result files matching %s were found, so publishing JUnit test results is being skipped." }, "_buildConfigMapping": { - "Default": "4.246.0", - "Node20-225": "4.246.1" + "Default": "4.246.2", + "Node20-225": "4.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV4/task.loc.json b/_generated/MavenV4/task.loc.json index b28993e10ff9..69a27b621312 100644 --- a/_generated/MavenV4/task.loc.json +++ b/_generated/MavenV4/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 0 + "Patch": 2 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ @@ -527,7 +527,7 @@ "NoTestResults": "ms-resource:loc.messages.NoTestResults" }, "_buildConfigMapping": { - "Default": "4.246.0", - "Node20-225": "4.246.1" + "Default": "4.246.2", + "Node20-225": "4.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV4_Node20/task.json b/_generated/MavenV4_Node20/task.json index 0ffbf028360b..364f6b5b5074 100644 --- a/_generated/MavenV4_Node20/task.json +++ b/_generated/MavenV4_Node20/task.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 1 + "Patch": 3 }, "releaseNotes": "Configuration of the SonarQube analysis was moved to the [SonarQube](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarqube) or [SonarCloud](https://marketplace.visualstudio.com/items?itemName=SonarSource.sonarcloud) extensions, in task `Prepare Analysis Configuration`", "demands": [ @@ -531,7 +531,7 @@ "NoTestResults": "No test result files matching %s were found, so publishing JUnit test results is being skipped." }, "_buildConfigMapping": { - "Default": "4.246.0", - "Node20-225": "4.246.1" + "Default": "4.246.2", + "Node20-225": "4.246.3" } } \ No newline at end of file diff --git a/_generated/MavenV4_Node20/task.loc.json b/_generated/MavenV4_Node20/task.loc.json index 97c59dc992ff..f9ad14270ad3 100644 --- a/_generated/MavenV4_Node20/task.loc.json +++ b/_generated/MavenV4_Node20/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 4, "Minor": 246, - "Patch": 1 + "Patch": 3 }, "releaseNotes": "ms-resource:loc.releaseNotes", "demands": [ @@ -531,7 +531,7 @@ "NoTestResults": "ms-resource:loc.messages.NoTestResults" }, "_buildConfigMapping": { - "Default": "4.246.0", - "Node20-225": "4.246.1" + "Default": "4.246.2", + "Node20-225": "4.246.3" } } \ No newline at end of file From 4cba1c9101e6b85708248f990be3b7def6c7cb79 Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 15:37:36 -0700 Subject: [PATCH 10/11] Fix for validation handling LocalPackages config --- ci/before-build-check-tasks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/before-build-check-tasks.js b/ci/before-build-check-tasks.js index 7afb472db8e1..aa4a4bff9a6f 100644 --- a/ci/before-build-check-tasks.js +++ b/ci/before-build-check-tasks.js @@ -30,7 +30,7 @@ function validateTaskSources( taskList ) { * @returns {Array} - array of messages with tasks which don't have versionmap file */ function validateGeneratedTasksExists( makeOptionJson ) { - const excludedKeys = ['tasks', 'taskResources']; + const excludedKeys = ['tasks', 'taskResources', 'LocalPackages']; // LocalPackages does not generate entries in verionmap file const errorList = []; for (const key in makeOptionJson) { From 5525f083969c7ab135ba29535d21db7a57168e18 Mon Sep 17 00:00:00 2001 From: Merlyn Oppenheim Date: Wed, 25 Sep 2024 17:18:14 -0700 Subject: [PATCH 11/11] Fix for validation state issue --- BuildConfigGen/EnsureUpdateModeVerifier.cs | 27 ++++++++++++++-------- BuildConfigGen/Program.cs | 3 ++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/BuildConfigGen/EnsureUpdateModeVerifier.cs b/BuildConfigGen/EnsureUpdateModeVerifier.cs index 5bfba98896f2..c19e71de6c1d 100644 --- a/BuildConfigGen/EnsureUpdateModeVerifier.cs +++ b/BuildConfigGen/EnsureUpdateModeVerifier.cs @@ -75,22 +75,31 @@ public IEnumerable GetVerifyErrors(bool skipContentCheck) public void CleanupTempFiles() { - int count = 0; - foreach (var f in RedirectedToTempl.Values) + try { - count++; - if (!tempsToKeep.Contains(f)) + int count = 0; + foreach (var f in RedirectedToTempl.Values) { - if (File.Exists(f)) + count++; + if (!tempsToKeep.Contains(f)) { - File.Delete(f); + if (File.Exists(f)) + { + File.Delete(f); + } } } - } - if (count > 0 && !verifyOnly) + if (count > 0 && !verifyOnly) + { + throw new Exception("Expected RedirectedToTemp to be empty when !verifyOnly"); + } + } + finally { - throw new Exception("Expected RedirectedToTemp to be empty when !verifyOnly"); + this.VerifyErrors.Clear(); + this.RedirectedToTempl.Clear(); + this.CopiedFilesToCheck.Clear(); } } diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index 3c821b0d5182..3a1d5e42e5c6 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -212,6 +212,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN } ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); + ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError("(global)", skipContentCheck: false); } foreach (var t in tasks) @@ -358,7 +359,7 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr Console.WriteLine(s); } - throw new Exception($"Updates needed, please run BuildConfigGen --task {task} --write-updates"); + throw new Exception($"Updates needed, please run npm make.js --task {task} "); } }