From 764b742f93225b421372870668307f6bd8a03ee3 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 25 May 2016 14:48:06 -0400 Subject: [PATCH 1/2] set working folder to system.defaultworkingdirectory for node handler. --- src/Agent.Worker/Handlers/NodeHandler.cs | 9 ++++++++- src/Agent.Worker/Variables.cs | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Agent.Worker/Handlers/NodeHandler.cs b/src/Agent.Worker/Handlers/NodeHandler.cs index 07c067b6e9..5ce79f1564 100644 --- a/src/Agent.Worker/Handlers/NodeHandler.cs +++ b/src/Agent.Worker/Handlers/NodeHandler.cs @@ -43,7 +43,14 @@ public async Task RunAsync() string workingDirectory = Data.WorkingDirectory; if (string.IsNullOrEmpty(workingDirectory)) { - workingDirectory = TaskDirectory; + if (!string.IsNullOrEmpty(ExecutionContext.Variables.System_DefaultWorkingDirectory)) + { + workingDirectory = ExecutionContext.Variables.System_DefaultWorkingDirectory; + } + else + { + workingDirectory = TaskDirectory; + } } ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); diff --git a/src/Agent.Worker/Variables.cs b/src/Agent.Worker/Variables.cs index 94dc085aba..e95f022a28 100644 --- a/src/Agent.Worker/Variables.cs +++ b/src/Agent.Worker/Variables.cs @@ -107,6 +107,7 @@ into maskHintGrouping public string Release_ReleaseUri { get { return Get(Constants.Variables.Release.ReleaseUri); } } public string System_CollectionId { get { return Get(Constants.Variables.System.CollectionId); } } public bool? System_Debug { get { return GetBoolean(Constants.Variables.System.Debug); } } + public string System_DefaultWorkingDirectory { get { return Get(Constants.Variables.System.DefaultWorkingDirectory); } } public string System_DefinitionId { get { return Get(Constants.Variables.System.DefinitionId); } } public bool? System_EnableAccessToken { get { return GetBoolean(Constants.Variables.System.EnableAccessToken); } } public string System_HostType { get { return Get(Constants.Variables.System.HostType); } } From 14fb9a109f702bd761022054b7dab47a2011869c Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 25 May 2016 17:12:08 -0400 Subject: [PATCH 2/2] update git source provider for packaged git. --- src/Agent.Worker/Build/GitCommandManager.cs | 115 +++++++++--------- src/Agent.Worker/Build/GitSourceProvider.cs | 57 +-------- .../Build/TfsGitSourceProvider.cs | 49 +------- src/Agent.Worker/Handlers/NodeHandler.cs | 2 +- src/Agent.Worker/Variables.cs | 1 + src/Misc/layoutbin/en-US/strings.json | 2 +- .../L0/Worker/Build/GitSourceProviderL0.cs | 3 + 7 files changed, 72 insertions(+), 157 deletions(-) diff --git a/src/Agent.Worker/Build/GitCommandManager.cs b/src/Agent.Worker/Build/GitCommandManager.cs index 4220fba849..69557b959a 100644 --- a/src/Agent.Worker/Build/GitCommandManager.cs +++ b/src/Agent.Worker/Build/GitCommandManager.cs @@ -6,17 +6,15 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using System.IO; namespace Microsoft.VisualStudio.Services.Agent.Worker.Build { [ServiceLocator(Default = typeof(GitCommandManager))] public interface IGitCommandManager : IAgentService { - string GitPath { get; set; } - - Version Version { get; set; } - - string GitHttpUserAgent { get; set; } + // setup git execution info, git location, version, useragent, execpath + Task LoadGitExecutionInfo(IExecutionContext context); // git init Task GitInit(IExecutionContext context, string repositoryPath); @@ -56,7 +54,7 @@ public interface IGitCommandManager : IAgentService // git config --unset-all Task GitConfigUnset(IExecutionContext context, string repositoryPath, string configKey); - + // git config gc.auto 0 Task GitDisableAutoGC(IExecutionContext context, string repositoryPath); @@ -66,20 +64,36 @@ public interface IGitCommandManager : IAgentService public class GitCommandManager : AgentService, IGitCommandManager { - private readonly Dictionary> _gitCommands = new Dictionary>(StringComparer.OrdinalIgnoreCase) + private string _gitHttpUserAgentEnv = null; + private string _gitPath = null; + private Version _version = null; + private string _gitExecPathEnv = null; + + public async Task LoadGitExecutionInfo(IExecutionContext context) { +#if OS_WINDOWS + _gitPath = Path.Combine(IOUtil.GetExternalsPath(), "git", "cmd", $"git{IOUtil.ExeExtension}"); +#else + _gitPath = Path.Combine(IOUtil.GetExternalsPath(), "git", "bin", $"git{IOUtil.ExeExtension}"); +#endif + if (string.IsNullOrEmpty(_gitPath) || !File.Exists(_gitPath)) { - "checkout", new Dictionary () - { - { new Version(1,8), "--force {0}" }, - { new Version(2,7), "--progress --force {0}" } - } + throw new Exception(StringUtil.Loc("GitNotFound")); } - }; - public string GitPath { get; set; } - public Version Version { get; set; } - public string GitHttpUserAgent { get; set; } + context.Debug($"Find git from agent's external directory: {_gitPath}."); + + _version = await GitVersion(context); + context.Debug($"Detect git version: {_version.ToString()}."); + + _gitHttpUserAgentEnv = $"git/{_version.ToString()} (vsts-agent-git/{Constants.Agent.Version})"; + context.Debug($"Set git useragent to: {_gitHttpUserAgentEnv}."); + +#if !OS_WINDOWS + _gitExecPathEnv = Path.Combine(IOUtil.GetExternalsPath(), "git", "libexec", "git-core"); + context.Debug($"Set git execpath to: {_gitExecPathEnv}"); +#endif + } // git init public async Task GitInit(IExecutionContext context, string repositoryPath) @@ -105,8 +119,7 @@ public async Task GitFetch(IExecutionContext context, string repositoryPath public async Task GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken) { context.Debug($"Checkout {committishOrBranchSpec}."); - string checkoutOption = GetCommandOption("checkout"); - return await ExecuteGitCommandAsync(context, repositoryPath, "checkout", StringUtil.Format(checkoutOption, committishOrBranchSpec), cancellationToken); + return await ExecuteGitCommandAsync(context, repositoryPath, "checkout", StringUtil.Format("--progress --force {0}", committishOrBranchSpec), cancellationToken); } // git clean -fdx @@ -225,6 +238,7 @@ public async Task GitVersion(IExecutionContext context) Version version = null; List outputStrings = new List(); int exitCode = await ExecuteGitCommandAsync(context, IOUtil.GetWorkPath(HostContext), "version", null, outputStrings); + context.Debug($"git version ouput: {string.Join(Environment.NewLine, outputStrings)}"); if (exitCode == 0) { // remove any empty line. @@ -248,32 +262,6 @@ public async Task GitVersion(IExecutionContext context) return version; } - private string GetCommandOption(string command) - { - if (string.IsNullOrEmpty(command)) - { - throw new ArgumentNullException("command"); - } - - if (!_gitCommands.ContainsKey(command)) - { - throw new NotSupportedException($"Unsupported git command: {command}"); - } - - Dictionary options = _gitCommands[command]; - foreach (var versionOption in options.OrderByDescending(o => o.Key)) - { - if (Version >= versionOption.Key) - { - return versionOption.Value; - } - } - - var earliestVersion = options.OrderByDescending(o => o.Key).Last(); - Trace.Info($"Fallback to version {earliestVersion.Key.ToString()} command option for git {command}."); - return earliestVersion.Value; - } - private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, CancellationToken cancellationToken = default(CancellationToken)) { string arg = StringUtil.Format($"{command} {options}").Trim(); @@ -290,13 +278,18 @@ private string GetCommandOption(string command) context.Output(message.Data); }; - Dictionary _userAgentEnv = new Dictionary(); - if (!string.IsNullOrEmpty(GitHttpUserAgent)) + Dictionary _gitEnv = new Dictionary(); + if (!string.IsNullOrEmpty(_gitHttpUserAgentEnv)) { - _userAgentEnv["GIT_HTTP_USER_AGENT"] = GitHttpUserAgent; + _gitEnv["GIT_HTTP_USER_AGENT"] = _gitHttpUserAgentEnv; } - return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, _userAgentEnv, cancellationToken); + if (!string.IsNullOrEmpty(_gitExecPathEnv)) + { + _gitEnv["GIT_EXEC_PATH"] = _gitExecPathEnv; + } + + return await processInvoker.ExecuteAsync(repoRoot, _gitPath, arg, _gitEnv, cancellationToken); } private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, IList output) @@ -327,13 +320,18 @@ private async Task ExecuteGitCommandAsync(IExecutionContext context, string } }; - Dictionary _userAgentEnv = new Dictionary(); - if (!string.IsNullOrEmpty(GitHttpUserAgent)) + Dictionary _gitEnv = new Dictionary(); + if (!string.IsNullOrEmpty(_gitHttpUserAgentEnv)) + { + _gitEnv["GIT_HTTP_USER_AGENT"] = _gitHttpUserAgentEnv; + } + + if (!string.IsNullOrEmpty(_gitExecPathEnv)) { - _userAgentEnv["GIT_HTTP_USER_AGENT"] = GitHttpUserAgent; + _gitEnv["GIT_EXEC_PATH"] = _gitExecPathEnv; } - return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, _userAgentEnv, default(CancellationToken)); + return await processInvoker.ExecuteAsync(repoRoot, _gitPath, arg, _gitEnv, default(CancellationToken)); } private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, string additionalCommandLine, CancellationToken cancellationToken) @@ -352,13 +350,18 @@ private async Task ExecuteGitCommandAsync(IExecutionContext context, string context.Output(message.Data); }; - Dictionary _userAgentEnv = new Dictionary(); - if (!string.IsNullOrEmpty(GitHttpUserAgent)) + Dictionary _gitEnv = new Dictionary(); + if (!string.IsNullOrEmpty(_gitHttpUserAgentEnv)) + { + _gitEnv["GIT_HTTP_USER_AGENT"] = _gitHttpUserAgentEnv; + } + + if (!string.IsNullOrEmpty(_gitExecPathEnv)) { - _userAgentEnv["GIT_HTTP_USER_AGENT"] = GitHttpUserAgent; + _gitEnv["GIT_EXEC_PATH"] = _gitExecPathEnv; } - return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, _userAgentEnv, cancellationToken); + return await processInvoker.ExecuteAsync(repoRoot, _gitPath, arg, _gitEnv, cancellationToken); } } } \ No newline at end of file diff --git a/src/Agent.Worker/Build/GitSourceProvider.cs b/src/Agent.Worker/Build/GitSourceProvider.cs index 1a63abd5a5..919267e22b 100644 --- a/src/Agent.Worker/Build/GitSourceProvider.cs +++ b/src/Agent.Worker/Build/GitSourceProvider.cs @@ -31,6 +31,8 @@ public virtual async Task GetSourceAsync(IExecutionContext executionContext, Ser ArgUtil.NotNull(endpoint, nameof(endpoint)); executionContext.Output($"Syncing repository: {endpoint.Name} (Git)"); + _gitCommandManager = HostContext.GetService(); + await _gitCommandManager.LoadGitExecutionInfo(executionContext); string targetPath = executionContext.Variables.Get(Constants.Variables.Build.SourcesDirectory); string sourceBranch = executionContext.Variables.Get(Constants.Variables.Build.SourceBranch); @@ -58,29 +60,6 @@ public virtual async Task GetSourceAsync(IExecutionContext executionContext, Ser Trace.Info($"checkoutSubmodules={checkoutSubmodules}"); Trace.Info($"exposeCred={exposeCred}"); - // ensure find full path to git exist, the version of the installed git is what we supported. - string gitPath = null; - if (!TryGetGitLocation(executionContext, out gitPath)) - { - throw new Exception(StringUtil.Loc("GitNotInstalled")); - } - Trace.Info($"Git path={gitPath}"); - - _gitCommandManager = HostContext.GetService(); - _gitCommandManager.GitPath = gitPath; - - Version gitVersion = await _gitCommandManager.GitVersion(executionContext); - if (gitVersion < _minSupportGitVersion) - { - throw new Exception(StringUtil.Loc("InstalledGitNotSupport", _minSupportGitVersion)); - } - Trace.Info($"Git version={gitVersion}"); - _gitCommandManager.Version = gitVersion; - - string customizeUserAgent = $"git/{gitVersion.ToString()} (vsts-agent-git/{Constants.Agent.Version})"; - Trace.Info($"Git useragent={customizeUserAgent}"); - _gitCommandManager.GitHttpUserAgent = customizeUserAgent; - // retrieve credential from endpoint. Uri repositoryUrl = endpoint.Url; if (!repositoryUrl.IsAbsoluteUri) @@ -309,38 +288,6 @@ public virtual async Task PostJobCleanupAsync(IExecutionContext executionContext await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin"); } - private bool TryGetGitLocation(IExecutionContext executionContext, out string gitPath) - { - //find git in %Path% - var whichTool = HostContext.GetService(); - gitPath = whichTool.Which("git"); - -#if OS_WINDOWS - //find in %ProgramFiles(x86)%\git\cmd if platform is Windows - if (string.IsNullOrEmpty(gitPath)) - { - string programFileX86 = Environment.GetEnvironmentVariable("ProgramFiles(x86)"); - if (!string.IsNullOrEmpty(programFileX86)) - { - gitPath = Path.Combine(programFileX86, "Git\\cmd\\git.exe"); - if (!File.Exists(gitPath)) - { - gitPath = null; - } - } - } -#endif - if (string.IsNullOrEmpty(gitPath)) - { - return false; - } - else - { - executionContext.Debug($"Find git installation path: {gitPath}."); - return true; - } - } - protected async Task IsRepositoryOriginUrlMatch(IExecutionContext context, string repositoryPath, Uri expectedRepositoryOriginUrl) { context.Debug($"Checking if the repo on {repositoryPath} matches the expected repository origin URL. expected Url: {expectedRepositoryOriginUrl.AbsoluteUri}"); diff --git a/src/Agent.Worker/Build/TfsGitSourceProvider.cs b/src/Agent.Worker/Build/TfsGitSourceProvider.cs index f58751e890..1b47a444d4 100644 --- a/src/Agent.Worker/Build/TfsGitSourceProvider.cs +++ b/src/Agent.Worker/Build/TfsGitSourceProvider.cs @@ -12,25 +12,17 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Build { public sealed class TfsGitSourceProvider : GitSourceProvider, ISourceProvider { - public override string RepositoryType => WellKnownRepositoryTypes.TfsGit; private readonly Dictionary _authHeaderCache = new Dictionary(); - private bool _supportAddAuthHeader = false; + + public override string RepositoryType => WellKnownRepositoryTypes.TfsGit; public override async Task GetSourceAsync(IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken) { Trace.Entering(); ArgUtil.NotNull(endpoint, nameof(endpoint)); - - string gitPath = null; - _supportAddAuthHeader = IsLocalGitSupportAddAuthHeader(out gitPath); - if (!_supportAddAuthHeader) - { - // use default git source provider handle credential which will embed credential into remote url - await base.GetSourceAsync(executionContext, endpoint, cancellationToken); - return; - } - executionContext.Output($"Syncing repository: {endpoint.Name} (TfsGit)"); + _gitCommandManager = HostContext.GetService(); + await _gitCommandManager.LoadGitExecutionInfo(executionContext); string targetPath = executionContext.Variables.Get(Constants.Variables.Build.SourcesDirectory); string sourceBranch = executionContext.Variables.Get(Constants.Variables.Build.SourceBranch); @@ -57,18 +49,6 @@ public override async Task GetSourceAsync(IExecutionContext executionContext, Se Trace.Info($"clean={clean}"); Trace.Info($"checkoutSubmodules={checkoutSubmodules}"); Trace.Info($"exposeCred={exposeCred}"); - Trace.Info($"Git path={gitPath}"); - - _gitCommandManager = HostContext.GetService(); - _gitCommandManager.GitPath = gitPath; - - Version gitVersion = await _gitCommandManager.GitVersion(executionContext); - Trace.Info($"Git version={gitVersion}"); - _gitCommandManager.Version = gitVersion; - - string customizeUserAgent = $"git/{gitVersion.ToString()} (vsts-agent-git/{Constants.Agent.Version})"; - Trace.Info($"Git useragent={customizeUserAgent}"); - _gitCommandManager.GitHttpUserAgent = customizeUserAgent; // retrieve credential from endpoint. Uri repositoryUrl = endpoint.Url; @@ -261,15 +241,9 @@ public override async Task GetSourceAsync(IExecutionContext executionContext, Se public override async Task PostJobCleanupAsync(IExecutionContext executionContext, ServiceEndpoint endpoint) { Trace.Entering(); - if (!_supportAddAuthHeader) - { - await base.PostJobCleanupAsync(executionContext, endpoint); - return; - } - ArgUtil.NotNull(endpoint, nameof(endpoint)); executionContext.Output($"Cleaning extra http auth header from repository: {endpoint.Name} (TfsGit)"); - + Uri repositoryUrl = endpoint.Url; string targetPath = executionContext.Variables.Get(Constants.Variables.Build.SourcesDirectory); @@ -298,18 +272,5 @@ public override async Task PostJobCleanupAsync(IExecutionContext executionContex } } } - - private bool IsLocalGitSupportAddAuthHeader(out string gitPath) - { - gitPath = string.Empty; - - // find portable git in externals -#if OS_WINDOWS - gitPath = Path.Combine(IOUtil.GetExternalsPath(), "git", "cmd", $"git{IOUtil.ExeExtension}"); -#else - gitPath = Path.Combine(IOUtil.GetExternalsPath(), "git", "bin", $"git{IOUtil.ExeExtension}"); -#endif - return !string.IsNullOrEmpty(gitPath) && File.Exists(gitPath); - } } } \ No newline at end of file diff --git a/src/Agent.Worker/Handlers/NodeHandler.cs b/src/Agent.Worker/Handlers/NodeHandler.cs index 5ce79f1564..20a151207a 100644 --- a/src/Agent.Worker/Handlers/NodeHandler.cs +++ b/src/Agent.Worker/Handlers/NodeHandler.cs @@ -49,7 +49,7 @@ public async Task RunAsync() } else { - workingDirectory = TaskDirectory; + workingDirectory = ExecutionContext.Variables.Agent_WorkFolder; } } diff --git a/src/Agent.Worker/Variables.cs b/src/Agent.Worker/Variables.cs index e95f022a28..e3d3a04894 100644 --- a/src/Agent.Worker/Variables.cs +++ b/src/Agent.Worker/Variables.cs @@ -87,6 +87,7 @@ into maskHintGrouping } public string Agent_BuildDirectory { get { return Get(Constants.Variables.Agent.BuildDirectory); } } + public string Agent_WorkFolder { get { return Get(Constants.Variables.Agent.WorkFolder); } } public TaskResult? Agent_JobStatus { get { return GetEnum(Constants.Variables.Agent.JobStatus); } set { Set(Constants.Variables.Agent.JobStatus, $"{value}"); } } public int? Build_BuildId { get { return GetInt(BuildWebApi.WellKnownBuildVariables.BuildId); } } public string Build_BuildUri { get { return Get(BuildWebApi.WellKnownBuildVariables.BuildUri); } } diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index 7098344ab2..84926df1da 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -75,7 +75,7 @@ "FileNotFound": "File not found: '{0}'", "FileUploadProgress": "Total file: {0} ---- Uploaded file: {1}", "GetSources": "Get Sources", - "GitNotInstalled": "Can't find installed git", + "GitNotFound": "Can't find git from agent external directory.", "GroupDoesNotExists": "Group: {0} does not Exist", "InstalledGitNotSupport": "The version of the installed Git is less than min-supported Git version {0}.", "InvalidBuildOrCoverageTool": "An internal error occurred. Details: Invalid build or code coverage tool provided. Build tool = '{0}'. Coverage tool = '{1}'. ", diff --git a/src/Test/L0/Worker/Build/GitSourceProviderL0.cs b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs index a6a0ea9c2c..d586c8d3f1 100644 --- a/src/Test/L0/Worker/Build/GitSourceProviderL0.cs +++ b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs @@ -18,6 +18,9 @@ public sealed class GitSourceProviderL0 private Mock GetDefaultGitCommandMock() { Mock _gitCommandManager = new Mock(); + _gitCommandManager + .Setup(x => x.LoadGitExecutionInfo(It.IsAny())) + .Returns(Task.CompletedTask); _gitCommandManager .Setup(x => x.GitInit(It.IsAny(), It.IsAny())) .Returns(Task.FromResult(0));