From 4eb2e6235644dbb586fdf0bf06f72ce783ca7149 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 30 Mar 2016 15:07:56 -0400 Subject: [PATCH] add L0 tests for git source provider. --- src/Agent.Worker/Build/GitCommandManager.cs | 355 ++++++++++++ src/Agent.Worker/Build/GitSourceProvider.cs | 529 ++++-------------- src/Misc/layoutbin/en-US/strings.json | 3 +- .../L0/Worker/Build/GitSourceProviderL0.cs | 353 ++++++++++++ 4 files changed, 819 insertions(+), 421 deletions(-) create mode 100644 src/Agent.Worker/Build/GitCommandManager.cs create mode 100644 src/Test/L0/Worker/Build/GitSourceProviderL0.cs diff --git a/src/Agent.Worker/Build/GitCommandManager.cs b/src/Agent.Worker/Build/GitCommandManager.cs new file mode 100644 index 0000000000..3b394bc53d --- /dev/null +++ b/src/Agent.Worker/Build/GitCommandManager.cs @@ -0,0 +1,355 @@ +using Microsoft.VisualStudio.Services.Agent.Util; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.VisualStudio.Services.Agent.Worker.Build +{ + [ServiceLocator(Default = typeof(GitCommandManager))] + public interface IGitCommandManager : IAgentService + { + string GitPath { get; set; } + + Version Version { get; set; } + + // git clone --progress --no-checkout + Task GitClone(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string username, string password, bool exposeCred, CancellationToken cancellationToken); + + // git fetch --tags --prune --progress origin [+refs/pull/*:refs/remote/pull/*] + Task GitFetch(IExecutionContext context, string repositoryPath, string remoteName, List refSpec, string username, string password, bool exposeCred, CancellationToken cancellationToken); + + // git checkout -f --progress + Task GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken); + + // git clean -fdx + Task GitClean(IExecutionContext context, string repositoryPath); + + // git reset --hard HEAD + Task GitReset(IExecutionContext context, string repositoryPath); + + // get remote set-url + Task GitRemoteSetUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl); + + // get remote set-url --push + Task GitRemoteSetPushUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl); + + // git submodule init + Task GitSubmoduleInit(IExecutionContext context, string repositoryPath); + + // git submodule update -f + Task GitSubmoduleUpdate(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken); + + // git config --get remote.origin.url + Task GitGetFetchUrl(IExecutionContext context, string repositoryPath); + + // git config --get-regexp submodule.*.url + Task> GitGetSubmoduleUrls(IExecutionContext context, string repoRoot); + + // git config + Task GitUpdateSubmoduleUrls(IExecutionContext context, string repositoryPath, Dictionary updateSubmoduleUrls); + + // git config gc.auto 0 + Task GitDisableAutoGC(IExecutionContext context, string repositoryPath); + + // git version + Task GitVersion(IExecutionContext context); + } + + public class GitCommandManager : AgentService, IGitCommandManager + { + private readonly Dictionary> _gitCommands = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { + "checkout", new Dictionary () + { + { new Version(1,8), "--force {0}" }, + { new Version(2,7), "--progress --force {0}" } + } + } + }; + + public string GitPath { get; set; } + public Version Version { get; set; } + + // git clone --progress --no-checkout + public async Task GitClone(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string username, string password, bool exposeCred, CancellationToken cancellationToken) + { + context.Debug($"Clone git repository: {repositoryUrl.AbsoluteUri} into: {repositoryPath}."); + string repoRootEscapeSpace = StringUtil.Format(@"""{0}""", repositoryPath.Replace(@"""", @"\""")); + return await ExecuteGitCommandAsync(context, repositoryPath, "clone", StringUtil.Format($"--progress --no-checkout {repositoryUrl.AbsoluteUri} {repoRootEscapeSpace}"), cancellationToken); + } + + // git fetch --tags --prune --progress origin [+refs/pull/*:refs/remote/pull/*] + public async Task GitFetch(IExecutionContext context, string repositoryPath, string remoteName, List refSpec, string username, string password, bool exposeCred, CancellationToken cancellationToken) + { + context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}."); + if (refSpec != null && refSpec.Count > 0) + { + refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList(); + } + + return await ExecuteGitCommandAsync(context, repositoryPath, "fetch", StringUtil.Format($"--tags --prune --progress {remoteName} {string.Join(" ", refSpec)}"), cancellationToken); + } + + // git checkout -f --progress + public async Task GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken = default(CancellationToken)) + { + context.Debug($"Checkout {committishOrBranchSpec}."); + string checkoutOption = GetCommandOption("checkout"); + return await ExecuteGitCommandAsync(context, repositoryPath, "checkout", StringUtil.Format(checkoutOption, committishOrBranchSpec), cancellationToken); + } + + // git clean -fdx + public async Task GitClean(IExecutionContext context, string repositoryPath) + { + context.Debug($"Delete untracked files/folders for repository at {repositoryPath}."); + return await ExecuteGitCommandAsync(context, repositoryPath, "clean", "-fdx"); + } + + // git reset --hard HEAD + public async Task GitReset(IExecutionContext context, string repositoryPath) + { + context.Debug($"Undo any changes to tracked files in the working tree for repository at {repositoryPath}."); + return await ExecuteGitCommandAsync(context, repositoryPath, "reset", "--hard HEAD"); + } + + // get remote set-url + public async Task GitRemoteSetUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) + { + context.Debug($"Set git fetch url to: {remoteUrl} for remote: {remoteName}."); + return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url {remoteName} {remoteUrl}")); + } + + // get remote set-url --push + public async Task GitRemoteSetPushUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) + { + context.Debug($"Set git push url to: {remoteUrl} for remote: {remoteName}."); + return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url --push {remoteName} {remoteUrl}")); + } + + // git submodule init + public async Task GitSubmoduleInit(IExecutionContext context, string repositoryPath) + { + context.Debug("Initialize the git submodules."); + return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "init"); + } + + // git submodule update -f + public async Task GitSubmoduleUpdate(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken = default(CancellationToken)) + { + context.Debug("Update the registered git submodules."); + return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "update -f", cancellationToken); + } + + // git config --get remote.origin.url + public async Task GitGetFetchUrl(IExecutionContext context, string repositoryPath) + { + context.Debug($"Inspect remote.origin.url for repository under {repositoryPath}"); + Uri fetchUrl = null; + + List outputStrings = new List(); + int exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", "--get remote.origin.url", outputStrings); + + if (exitCode != 0) + { + context.Warning($"'git config --get remote.origin.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'"); + } + else + { + // remove empty strings + outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); + if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First())) + { + string remoteFetchUrl = outputStrings.First(); + if (Uri.IsWellFormedUriString(remoteFetchUrl, UriKind.Absolute)) + { + context.Debug($"Get remote origin fetch url from git config: {remoteFetchUrl}"); + fetchUrl = new Uri(remoteFetchUrl); + } + else + { + context.Debug($"The Origin fetch url from git config: {remoteFetchUrl} is not a absolute well formed url."); + } + } + else + { + context.Debug($"Unable capture git remote fetch uri from 'git config --get remote.origin.url' command's output, the command's output is not expected: {string.Join(Environment.NewLine, outputStrings)}."); + } + } + + return fetchUrl; + } + + // git config --get-regexp submodule.*.url + public async Task> GitGetSubmoduleUrls(IExecutionContext context, string repoRoot) + { + context.Debug($"Inspect all submodule..url for submodules under {repoRoot}"); + + Dictionary submoduleUrls = new Dictionary(StringComparer.OrdinalIgnoreCase); + + List outputStrings = new List(); + int exitCode = await ExecuteGitCommandAsync(context, repoRoot, "config", "--get-regexp submodule.?*.url", outputStrings); + + if (exitCode != 0) + { + context.Debug($"'git config --get-regexp submodule.?*.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'"); + } + else + { + // remove empty strings + outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); + foreach (var urlString in outputStrings) + { + context.Debug($"Potential git submodule name and fetch url: {urlString}."); + string[] submoduleUrl = urlString.Split(new Char[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries); + if (submoduleUrl.Length == 2 && Uri.IsWellFormedUriString(submoduleUrl[1], UriKind.Absolute)) + { + submoduleUrls[submoduleUrl[0]] = new Uri(submoduleUrl[1]); + } + else + { + context.Debug($"Can't parse git submodule name and submodule fetch url from output: '{urlString}'."); + } + } + } + + return submoduleUrls; + } + + // git config + public async Task GitUpdateSubmoduleUrls(IExecutionContext context, string repositoryPath, Dictionary updateSubmoduleUrls) + { + context.Debug("Update all submodule..url with credential embeded url."); + + int overallExitCode = 0; + foreach (var submodule in updateSubmoduleUrls) + { + Int32 exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"{submodule.Key} {submodule.Value.ToString()}")); + if (exitCode != 0) + { + context.Debug($"Unable update: {submodule.Key}."); + overallExitCode = exitCode; + } + } + + return overallExitCode; + } + + // git config gc.auto 0 + public async Task GitDisableAutoGC(IExecutionContext context, string repositoryPath) + { + context.Debug("Disable git auto garbage collection."); + return await ExecuteGitCommandAsync(context, repositoryPath, "config", "gc.auto 0"); + } + + // git version + public async Task GitVersion(IExecutionContext context) + { + context.Debug("Get git version."); + Version version = null; + List outputStrings = new List(); + int exitCode = await ExecuteGitCommandAsync(context, IOUtil.GetWorkPath(HostContext), "version", null, outputStrings); + if (exitCode == 0) + { + // remove any empty line. + outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); + if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First())) + { + string verString = outputStrings.First(); + // we might only interested about major.minor version + Regex verRegex = new Regex("\\d+\\.\\d+", RegexOptions.IgnoreCase); + var matchResult = verRegex.Match(verString); + if (matchResult.Success && !string.IsNullOrEmpty(matchResult.Value)) + { + if (!Version.TryParse(matchResult.Value, out version)) + { + version = null; + } + } + } + } + + return version; + } + + private string GetCommandOption(string command) + { + if (string.IsNullOrEmpty(command)) + { + throw new ArgumentNullException("command"); + } + + if (!_gitCommands.ContainsKey(command)) + { + throw new NotSupportedException($"Unsupported git command: {command}"); + } + + Dictionary options = _gitCommands[command]; + foreach (var versionOption in options.OrderByDescending(o => o.Key)) + { + if (Version >= versionOption.Key) + { + return versionOption.Value; + } + } + + var earliestVersion = options.OrderByDescending(o => o.Key).Last(); + Trace.Info($"Fallback to version {earliestVersion.Key.ToString()} command option for git {command}."); + return earliestVersion.Value; + } + + private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, CancellationToken cancellationToken = default(CancellationToken)) + { + string arg = StringUtil.Format($"{command} {options}").Trim(); + context.Command($"git {arg}"); + + var processInvoker = HostContext.CreateService(); + processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message) + { + context.Output(message.Data); + }; + + processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message) + { + context.Output(message.Data); + }; + + return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, null, cancellationToken); + } + + private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, IList output) + { + string arg = StringUtil.Format($"{command} {options}").Trim(); + context.Command($"git {arg}"); + + if (output == null) + { + output = new List(); + } + + object outputLock = new object(); + var processInvoker = HostContext.CreateService(); + processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message) + { + lock (outputLock) + { + output.Add(message.Data); + } + }; + + processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message) + { + lock (outputLock) + { + output.Add(message.Data); + } + }; + + return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, null, default(CancellationToken)); + } + } +} \ No newline at end of file diff --git a/src/Agent.Worker/Build/GitSourceProvider.cs b/src/Agent.Worker/Build/GitSourceProvider.cs index 28ecaf17a2..83246c74db 100644 --- a/src/Agent.Worker/Build/GitSourceProvider.cs +++ b/src/Agent.Worker/Build/GitSourceProvider.cs @@ -1,16 +1,11 @@ using Microsoft.TeamFoundation.Build.WebApi; -using Microsoft.VisualStudio.Services.Agent; -using System; using Microsoft.TeamFoundation.DistributedTask.WebApi; -using System.Threading; -using System.Threading.Tasks; using Microsoft.VisualStudio.Services.Agent.Util; -using System.IO; -using System.Globalization; +using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text.RegularExpressions; +using System.IO; +using System.Threading; +using System.Threading.Tasks; namespace Microsoft.VisualStudio.Services.Agent.Worker.Build { @@ -25,19 +20,7 @@ public class GitSourceProvider : SourceProvider, ISourceProvider private static Version _minSupportGitVersion = new Version(1, 8); private readonly Dictionary _credentialUrlCache = new Dictionary(); - private readonly Dictionary> _gitCommands = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { - "checkout", new Dictionary () - { - { new Version(1,8), "--force {0}" }, - { new Version(2,7), "--progress --force {0}" } - } - } - }; - - private string _gitPath = null; - private Version _gitVersion = null; + private IGitCommandManager _gitCommandManager; public override string RepositoryType => WellKnownRepositoryTypes.Git; @@ -58,7 +41,6 @@ public async Task GetSourceAsync(IExecutionContext executionContext, ServiceEndp clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]); } - bool checkoutSubmodules = false; if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules)) { @@ -75,15 +57,27 @@ public async Task GetSourceAsync(IExecutionContext executionContext, ServiceEndp Trace.Info($"checkoutSubmodules={checkoutSubmodules}"); Trace.Info($"exposeCred={exposeCred}"); - // Find full path to git, get version of the installed git. - if (!await TrySetGitInstallationInfo(executionContext)) + // ensure find full path to git exist, the version of the installed git is what we supported. + string gitPath = null; + if (!TryGetGitLocation(executionContext, out gitPath)) + { + throw new Exception(StringUtil.Loc("GitNotInstalled")); + } + Trace.Info($"Git path={gitPath}"); + + _gitCommandManager = HostContext.GetService(); + _gitCommandManager.GitPath = gitPath; + + Version gitVersion = await _gitCommandManager.GitVersion(executionContext); + if (gitVersion < _minSupportGitVersion) { throw new Exception(StringUtil.Loc("InstalledGitNotSupport", _minSupportGitVersion)); } + Trace.Info($"Git version={gitVersion}"); + _gitCommandManager.Version = gitVersion; + // sync source await SyncAndCheckout(executionContext, endpoint, targetPath, clean, sourceBranch, sourceVersion, checkoutSubmodules, exposeCred, cancellationToken); - - return; } public async Task PostJobCleanupAsync(IExecutionContext executionContext, ServiceEndpoint endpoint) @@ -109,63 +103,52 @@ public string GetLocalPath(ServiceEndpoint endpoint, string path) return path; } - private async Task TrySetGitInstallationInfo(IExecutionContext executionContext) + private bool TryGetGitLocation(IExecutionContext executionContext, out string gitPath) { //find git in %Path% var whichTool = HostContext.GetService(); - _gitPath = whichTool.Which("git"); + gitPath = whichTool.Which("git"); #if OS_WINDOWS //find in %ProgramFiles(x86)%\git\cmd if platform is Windows - if (string.IsNullOrEmpty(_gitPath)) + if (string.IsNullOrEmpty(gitPath)) { string programFileX86 = Environment.GetEnvironmentVariable("ProgramFiles(x86)"); if (!string.IsNullOrEmpty(programFileX86)) { - _gitPath = Path.Combine(programFileX86, "Git\\cmd\\git.exe"); - if (!File.Exists(_gitPath)) + gitPath = Path.Combine(programFileX86, "Git\\cmd\\git.exe"); + if (!File.Exists(gitPath)) { - _gitPath = null; + gitPath = null; } } } #endif - if (string.IsNullOrEmpty(_gitPath)) - { - return false; - } - else - { - executionContext.Debug($"Find git installation path: {_gitPath}."); - } - - _gitVersion = await GitVersion(executionContext, _gitPath); - if (_gitVersion == null || _gitVersion < _minSupportGitVersion) + if (string.IsNullOrEmpty(gitPath)) { return false; } else { - executionContext.Debug($"The version of the installed git is: {_gitVersion}."); + executionContext.Debug($"Find git installation path: {gitPath}."); + return true; } - - return true; } private async Task SyncAndCheckout( IExecutionContext context, ServiceEndpoint endpoint, string targetPath, - bool clean, - string sourceBranch, + bool clean, + string sourceBranch, string sourceVersion, - bool checkoutSubmodules, - bool exposeCred, + bool checkoutSubmodules, + bool exposeCred, CancellationToken cancellationToken = default(CancellationToken)) { Trace.Entering(); cancellationToken.ThrowIfCancellationRequested(); - Int32 gitCommandExitCode; + int gitCommandExitCode; // retrieve credential from endpoint. Uri repositoryUrl = endpoint.Url; @@ -211,8 +194,7 @@ private async Task SyncAndCheckout( if (!await IsRepositoryOriginUrlMatch(context, targetPath, repositoryUrl)) { // Delete source folder - // TODO: add IO Util Delete() handle cancellation and exception - Directory.Delete(targetPath, true); + IOUtil.DeleteDirectory(targetPath, cancellationToken); } else { @@ -224,14 +206,14 @@ private async Task SyncAndCheckout( Boolean softClean = false; // git clean -fdx // git reset --hard HEAD - gitCommandExitCode = await GitClean(context, targetPath); + gitCommandExitCode = await _gitCommandManager.GitClean(context, targetPath); if (gitCommandExitCode != 0) { context.Debug($"'git clean -fdx' failed with exit code {gitCommandExitCode}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build."); } else { - gitCommandExitCode = await GitReset(context, targetPath); + gitCommandExitCode = await _gitCommandManager.GitReset(context, targetPath); if (gitCommandExitCode != 0) { context.Debug($"'git reset --hard HEAD' failed with exit code {gitCommandExitCode}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build."); @@ -258,16 +240,44 @@ private async Task SyncAndCheckout( Directory.CreateDirectory(targetPath); } + // inject credential into fetch url + context.Debug("Inject credential into git remote url."); + Uri urlWithCred = null; + urlWithCred = GetCredentialEmbeddedRepoUrl(repositoryUrl, username, password); + // if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state. // we will run git fetch to update the repo. if (Directory.Exists(Path.Combine(targetPath, ".git"))) { + // disable git auto gc + int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(context, targetPath); + if (exitCode_disableGC != 0) + { + context.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching."); + } + + // inject credential into fetch url + context.Debug("Inject credential into git remote fetch url."); + int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(context, targetPath, "origin", urlWithCred.AbsoluteUri); + if (exitCode_seturl != 0) + { + throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}"); + } + + // inject credential into push url + context.Debug("Inject credential into git remote push url."); + exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(context, targetPath, "origin", urlWithCred.AbsoluteUri); + if (exitCode_seturl != 0) + { + throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}"); + } + // If this is a build for a pull request, then include // the pull request reference as an additional ref. string fetchSpec = IsPullRequest(sourceBranch) ? StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)) : null; context.Progress(0, "Starting fetch..."); - gitCommandExitCode = await GitFetch(context, targetPath, repositoryUrl, "origin", new List() { fetchSpec }, username, password, exposeCred, cancellationToken); + gitCommandExitCode = await _gitCommandManager.GitFetch(context, targetPath, "origin", new List() { fetchSpec }, username, password, exposeCred, cancellationToken); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git fetch failed with exit code: {gitCommandExitCode}"); @@ -276,7 +286,7 @@ private async Task SyncAndCheckout( else { context.Progress(0, "Starting clone..."); - gitCommandExitCode = await GitClone(context, targetPath, repositoryUrl, username, password, exposeCred, cancellationToken); + gitCommandExitCode = await _gitCommandManager.GitClone(context, targetPath, urlWithCred, username, password, exposeCred, cancellationToken); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git clone failed with exit code: {gitCommandExitCode}"); @@ -289,7 +299,7 @@ private async Task SyncAndCheckout( context.Progress(76, $"Starting fetch pull request ref... {fetchSpec}"); context.Output("Starting fetch pull request ref"); - gitCommandExitCode = await GitFetch(context, targetPath, repositoryUrl, "origin", new List() { fetchSpec }, username, password, exposeCred, cancellationToken); + gitCommandExitCode = await _gitCommandManager.GitFetch(context, targetPath, "origin", new List() { fetchSpec }, username, password, exposeCred, cancellationToken); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git fetch failed with exit code: {gitCommandExitCode}"); @@ -297,18 +307,26 @@ private async Task SyncAndCheckout( } } + if (!exposeCred) + { + // remove cached credential from origin's fetch/push url. + await RemoveCachedCredential(context, targetPath, repositoryUrl, "origin"); + } + // Checkout // delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time. string lockFile = Path.Combine(targetPath, ".git\\index.lock"); - try - { - // TODO: IOUtil.FileDelete() - File.Delete(lockFile); - } - catch (Exception ex) + if (File.Exists(lockFile)) { - context.Debug($"Unable to delete the index.lock file: {lockFile}"); - context.Debug(ex.ToString()); + try + { + File.Delete(lockFile); + } + catch (Exception ex) + { + context.Debug($"Unable to delete the index.lock file: {lockFile}"); + context.Debug(ex.ToString()); + } } // sourceToBuild is used for checkout @@ -327,7 +345,7 @@ private async Task SyncAndCheckout( } // Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw) - gitCommandExitCode = await GitCheckout(context, targetPath, sourcesToBuild, cancellationToken); + gitCommandExitCode = await _gitCommandManager.GitCheckout(context, targetPath, sourcesToBuild, cancellationToken); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git checkout failed with exit code: {gitCommandExitCode}"); @@ -337,7 +355,7 @@ private async Task SyncAndCheckout( if (checkoutSubmodules) { context.Progress(90, "Updating submodules..."); - gitCommandExitCode = await GitSubmoduleInit(context, targetPath); + gitCommandExitCode = await _gitCommandManager.GitSubmoduleInit(context, targetPath); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git submodule init failed with exit code: {gitCommandExitCode}"); @@ -350,7 +368,7 @@ private async Task SyncAndCheckout( // GitUpdateSubmoduleUrls(m_rootPath, submoduleUrls); context.Command("git submodule update"); - gitCommandExitCode = await GitSubmoduleUpdate(context, targetPath, cancellationToken); + gitCommandExitCode = await _gitCommandManager.GitSubmoduleUpdate(context, targetPath, cancellationToken); if (gitCommandExitCode != 0) { throw new InvalidOperationException($"Git submodule update failed with exit code: {gitCommandExitCode}"); @@ -358,254 +376,38 @@ private async Task SyncAndCheckout( } } - // git clone --progress --no-checkout - private async Task GitClone(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string username, string password, bool exposeCred, CancellationToken cancellationToken) - { - context.Debug($"Clone git repository: {repositoryUrl.AbsoluteUri} into: {repositoryPath}."); - - // inject credential into fetch url - context.Debug("Inject credential into git remote url."); - Uri urlWithCred = null; - urlWithCred = GetCredentialEmbeddedRepoUrl(repositoryUrl, username, password); - - string repoRootEscapeSpace = StringUtil.Format(@"""{0}""", repositoryPath.Replace(@"""", @"\""")); - Int32 exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "clone", StringUtil.Format($"--progress --no-checkout {urlWithCred.AbsoluteUri} {repoRootEscapeSpace}"), cancellationToken); - - if (!exposeCred) - { - // remove cached credential from origin's fetch/push url. - await RemoveCachedCredential(context, repositoryPath, repositoryUrl, "origin"); - } - - return exitCode; - } - - // git fetch --tags --prune --progress origin [+refs/pull/*:refs/remote/pull/*] - private async Task GitFetch(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string remoteName, List refSpec, string username, string password, bool exposeCred, CancellationToken cancellationToken) + private async Task IsRepositoryOriginUrlMatch(IExecutionContext context, string repositoryPath, Uri expectedRepositoryOriginUrl) { - if (refSpec != null && refSpec.Count > 0) - { - refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList(); - } - - context.Debug($"Fetch git repository: {repositoryUrl.AbsoluteUri} remote: {remoteName}."); - - // disable git auto gc - Int32 exitCode_disableGC = await GitDisableAutoGC(context, repositoryPath); - if (exitCode_disableGC != 0) - { - context.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching."); - } - - Uri urlWithCred = null; - urlWithCred = GetCredentialEmbeddedRepoUrl(repositoryUrl, username, password); - - // inject credential into fetch url - context.Debug("Inject credential into git remote fetch url."); - Int32 exitCode_seturl = await GitRemoteSetUrl(context, repositoryPath, remoteName, urlWithCred.AbsoluteUri); - if (exitCode_seturl != 0) - { - throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}"); - } - - // inject credential into push url - context.Debug("Inject credential into git remote push url."); - exitCode_seturl = await GitRemoteSetPushUrl(context, repositoryPath, remoteName, urlWithCred.AbsoluteUri); - if (exitCode_seturl != 0) - { - throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}"); - } - - Int32 exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", StringUtil.Format($"--tags --prune --progress {remoteName} {string.Join(" ", refSpec)}"), cancellationToken); - - if (!exposeCred) + context.Debug($"Checking if the repo on {repositoryPath} matches the expected repository origin URL. expected Url: {expectedRepositoryOriginUrl.AbsoluteUri}"); + if (!Directory.Exists(Path.Combine(repositoryPath, ".git"))) { - // remove cached credential from origin's fetch/push url. - await RemoveCachedCredential(context, repositoryPath, repositoryUrl, "origin"); + // There is no repo directory + context.Debug($"Repository is not found since '.git' directory does not exist under. {repositoryPath}"); + return false; } - return exitCode; - } - - // git checkout -f --progress - private async Task GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken = default(CancellationToken)) - { - context.Debug($"Checkout {committishOrBranchSpec}."); - string checkoutOption = GetCommandOption("checkout", _gitVersion); - return await ExecuteGitCommandAsync(context, repositoryPath, "checkout", StringUtil.Format(checkoutOption, committishOrBranchSpec), cancellationToken); - } - - // git clean -fdx - private async Task GitClean(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken = default(CancellationToken)) - { - context.Debug($"Delete untracked files/folders for repository at {repositoryPath}."); - return await ExecuteGitCommandAsync(context, repositoryPath, "clean", "-fdx", cancellationToken); ; - } - - // git reset --hard HEAD - private async Task GitReset(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken = default(CancellationToken)) - { - context.Debug($"Undo any changes to tracked files in the working tree for repository at {repositoryPath}."); - return await ExecuteGitCommandAsync(context, repositoryPath, "reset", "--hard HEAD", cancellationToken); - } - - // get remote set-url - private async Task GitRemoteSetUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) - { - context.Debug($"Set git fetch url to: {remoteUrl} for remote: {remoteName}."); - return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url {remoteName} {remoteUrl}")); - } - - // get remote set-url --push - private async Task GitRemoteSetPushUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) - { - context.Debug($"Set git push url to: {remoteUrl} for remote: {remoteName}."); - return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url --push {remoteName} {remoteUrl}")); - } - - // git submodule init - private async Task GitSubmoduleInit(IExecutionContext context, string repositoryPath) - { - context.Debug("Initialize the git submodules."); - return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "init"); - } - - // git submodule update -f - private async Task GitSubmoduleUpdate(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken = default(CancellationToken)) - { - context.Debug("Update the registered git submodules."); - return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "update -f", cancellationToken); - } - - // git config --get remote.origin.url - private async Task GitGetFetchUrl(IExecutionContext context, string repositoryPath) - { - context.Debug($"Inspect remote.origin.url for repository under {repositoryPath}"); - Uri fetchUrl = null; - - List outputStrings = new List(); - int exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", "--get remote.origin.url", outputStrings); + Uri remoteUrl; + remoteUrl = await _gitCommandManager.GitGetFetchUrl(context, repositoryPath); - if (exitCode != 0) - { - context.Warning($"'git config --get remote.origin.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'"); - } - else + if (remoteUrl == null) { - // remove empty strings - outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); - if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First())) - { - string remoteFetchUrl = outputStrings.First(); - if (Uri.IsWellFormedUriString(remoteFetchUrl, UriKind.Absolute)) - { - context.Debug($"Get remote origin fetch url from git config: {remoteFetchUrl}"); - fetchUrl = new Uri(remoteFetchUrl); - } - else - { - context.Debug($"The Origin fetch url from git config: {remoteFetchUrl} is not a absolute well formed url."); - } - } - else - { - context.Debug($"Unable capture git remote fetch uri from 'git config --get remote.origin.url' command's output, the command's output is not expected: {string.Join(Environment.NewLine, outputStrings)}."); - } + // origin fetch url not found. + context.Debug("Repository remote origin fetch url is empty."); + return false; } - return fetchUrl; - } - - // git config --get-regexp submodule.*.url - private async Task> GitGetSubmoduleUrls(IExecutionContext context, string repoRoot) - { - context.Debug($"Inspect all submodule..url for submodules under {repoRoot}"); - - Dictionary submoduleUrls = new Dictionary(StringComparer.OrdinalIgnoreCase); - - List outputStrings = new List(); - int exitCode = await ExecuteGitCommandAsync(context, repoRoot, "config", "--get-regexp submodule.?*.url", outputStrings); - - if (exitCode != 0) + context.Debug($"Repository remote origin fetch url is {remoteUrl}"); + // compare the url passed in with the remote url found + if (expectedRepositoryOriginUrl.Equals(remoteUrl)) { - context.Debug($"'git config --get-regexp submodule.?*.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'"); + context.Debug("URLs match."); + return true; } else { - // remove empty strings - outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); - foreach (var urlString in outputStrings) - { - context.Debug($"Potential git submodule name and fetch url: {urlString}."); - string[] submoduleUrl = urlString.Split(new Char[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries); - if (submoduleUrl.Length == 2 && Uri.IsWellFormedUriString(submoduleUrl[1], UriKind.Absolute)) - { - submoduleUrls[submoduleUrl[0]] = new Uri(submoduleUrl[1]); - } - else - { - context.Debug($"Can't parse git submodule name and submodule fetch url from output: '{urlString}'."); - } - } - } - - return submoduleUrls; - } - - // git config - private async Task GitUpdateSubmoduleUrls(IExecutionContext context, string repositoryPath, Dictionary updateSubmoduleUrls) - { - context.Debug("Update all submodule..url with credential embeded url."); - - int overallExitCode = 0; - foreach (var submodule in updateSubmoduleUrls) - { - Int32 exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"{submodule.Key} {submodule.Value.ToString()}")); - if (exitCode != 0) - { - context.Debug($"Unable update: {submodule.Key}."); - overallExitCode = exitCode; - } - } - - return overallExitCode; - } - - // git config gc.auto 0 - private async Task GitDisableAutoGC(IExecutionContext context, string repositoryPath) - { - context.Debug("Disable git auto garbage collection."); - return await ExecuteGitCommandAsync(context, repositoryPath, "config", "gc.auto 0"); - } - - // git version - public async Task GitVersion(IExecutionContext context, string gitPath) - { - context.Debug("Get git version."); - Version version = null; - List outputStrings = new List(); - int exitCode = await ExecuteGitCommandAsync(context, IOUtil.GetWorkPath(HostContext), "version", null, outputStrings); - if (exitCode == 0) - { - // remove any empty line. - outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList(); - if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First())) - { - string verString = outputStrings.First(); - // we might only interested about major.minor version - Regex verRegex = new Regex("\\d+\\.\\d+", RegexOptions.IgnoreCase); - var matchResult = verRegex.Match(verString); - if (matchResult.Success && !string.IsNullOrEmpty(matchResult.Value)) - { - if (!Version.TryParse(matchResult.Value, out version)) - { - version = null; - } - } - } + context.Debug($"The remote.origin.url of the repository under root folder '{repositoryPath}' doesn't matches source repository url."); + return false; } - - return version; } private Uri GetCredentialEmbeddedRepoUrl(Uri repositoryUrl, string username, string password) @@ -665,48 +467,14 @@ private Uri GetCredentialEmbeddedRepoUrl(Uri repositoryUrl, string username, str } } - private async Task IsRepositoryOriginUrlMatch(IExecutionContext context, string repositoryPath, Uri expectedRepositoryOriginUrl) - { - context.Debug($"Checking if the repo on {repositoryPath} matches the expected repository origin URL. expected Url: {expectedRepositoryOriginUrl.AbsoluteUri}"); - if (!Directory.Exists(Path.Combine(repositoryPath, ".git"))) - { - // There is no repo directory - context.Debug($"Repository is not found since '.git' directory does not exist under. {repositoryPath}"); - return false; - } - - Uri remoteUrl; - remoteUrl = await GitGetFetchUrl(context, repositoryPath); - - if (remoteUrl == null) - { - // origin fetch url not found. - context.Debug("Repository remote origin fetch url is empty."); - return false; - } - - context.Debug($"Repository remote origin fetch url is {remoteUrl}"); - // compare the url passed in with the remote url found - if (expectedRepositoryOriginUrl.Equals(remoteUrl)) - { - context.Debug("URLs match."); - return true; - } - else - { - context.Debug($"The remote.origin.url of the repository under root folder '{repositoryPath}' doesn't matches source repository url."); - return false; - } - } - private async Task RemoveCachedCredential(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string remoteName) { //remove credential from fetch url context.Debug("Remove injected credential from git remote fetch url."); - Int32 exitCode_seturl = await GitRemoteSetUrl(context, repositoryPath, remoteName, repositoryUrl.AbsoluteUri); + int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(context, repositoryPath, remoteName, repositoryUrl.AbsoluteUri); context.Debug("Remove injected credential from git remote push url."); - Int32 exitCode_setpushurl = await GitRemoteSetPushUrl(context, repositoryPath, remoteName, repositoryUrl.AbsoluteUri); + int exitCode_setpushurl = await _gitCommandManager.GitRemoteSetPushUrl(context, repositoryPath, remoteName, repositoryUrl.AbsoluteUri); if (exitCode_seturl != 0 || exitCode_setpushurl != 0) { @@ -727,85 +495,6 @@ private async Task RemoveCachedCredential(IExecutionContext context, string repo } } - private string GetCommandOption(string command, Version version) - { - if (string.IsNullOrEmpty(command)) - { - throw new ArgumentNullException("command"); - } - - if (version < _minSupportGitVersion) - { - throw new NotSupportedException($"MinSupported git version is {_minSupportGitVersion}, your request version is {version}"); - } - - if (!_gitCommands.ContainsKey(command)) - { - throw new NotSupportedException($"Unsupported git command: {command}"); - } - - Dictionary options = _gitCommands[command]; - foreach (var versionOption in options.OrderByDescending(o => o.Key)) - { - if (version >= versionOption.Key) - { - return versionOption.Value; - } - } - - throw new NotSupportedException($"Can't find supported git command option for command: {command}, version: {version}."); - } - - private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, CancellationToken cancellationToken = default(CancellationToken)) - { - string arg = StringUtil.Format($"{command} {options}").Trim(); - context.Command($"git {arg}"); - - var processInvoker = HostContext.CreateService(); - processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message) - { - context.Output(message.Data); - }; - - processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message) - { - context.Output(message.Data); - }; - - return await processInvoker.ExecuteAsync(repoRoot, _gitPath, arg, null, cancellationToken); - } - - private async Task ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, IList output) - { - string arg = StringUtil.Format($"{command} {options}").Trim(); - context.Command($"git {arg}"); - - if (output == null) - { - output = new List(); - } - - object outputLock = new object(); - var processInvoker = HostContext.CreateService(); - processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message) - { - lock(outputLock) - { - output.Add(message.Data); - } - }; - - processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message) - { - lock (outputLock) - { - output.Add(message.Data); - } - }; - - return await processInvoker.ExecuteAsync(repoRoot, _gitPath, arg, null, default(CancellationToken)); - } - private bool IsPullRequest(string sourceBranch) { return !string.IsNullOrEmpty(sourceBranch) && diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index 5cbe6e8810..f1a97f64e1 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -50,8 +50,9 @@ "FailedToReplaceAgent": "Failed to replace the agent. Try again or ctrl-c to quit", "FileUploadProgress": "Total file: {0} ---- Uploaded file: {1}", "GetSources": "Get Sources", + "GitNotInstalled": "Can't find installed git", "GroupDoesNotExists": "Group: {0} does not Exist", - "InstalledGitNotSupport": "Unable to find installed Git in Path or the version of the installed Git is less than min-supported Git version {0}.", + "InstalledGitNotSupport": "The version of the installed Git is less than min-supported Git version {0}.", "InvalidConfigFor0TerminatingUnattended": "Invalid configuration provided for {0}. Terminating unattended configuration.", "InvalidGroupName": "Invalid Group Name - {0}", "InvalidMember": "A new member could not be added to a local group because the member has the wrong account type. If you are configuring on a domain controller, built-in machine accounts cannot be added to local groups. You must use a domain user account instead", diff --git a/src/Test/L0/Worker/Build/GitSourceProviderL0.cs b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs new file mode 100644 index 0000000000..9fa6bc9a01 --- /dev/null +++ b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs @@ -0,0 +1,353 @@ +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.TeamFoundation.DistributedTask.WebApi; +using Microsoft.VisualStudio.Services.Agent.Util; +using Microsoft.VisualStudio.Services.Agent.Worker; +using Microsoft.VisualStudio.Services.Agent.Worker.Build; +using Moq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.VisualStudio.Services.Agent.Tests.Worker.Build +{ + public sealed class GitSourceProviderL0 + { + private Mock GetDefaultGitCommandMock() + { + Mock _gitCommandManager = new Mock(); + _gitCommandManager + .Setup(x => x.GitClone(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitFetch(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitCheckout(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitClean(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitReset(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitRemoteSetUrl(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitRemoteSetPushUrl(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitSubmoduleInit(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitSubmoduleUpdate(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitGetFetchUrl(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(new Uri("https://github.com/Microsoft/vsts-agent"))); + _gitCommandManager + .Setup(x => x.GitDisableAutoGC(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(0)); + _gitCommandManager + .Setup(x => x.GitVersion(It.IsAny())) + .Returns(Task.FromResult(new Version(2, 7))); + + return _gitCommandManager; + } + + private Mock GetTestExecutionContext(TestHostContext tc, string sourceFolder, string sourceBranch, string sourceVersion, bool enableAuth) + { + var trace = tc.GetTrace(); + var executionContext = new Mock(); + List warnings; + executionContext + .Setup(x => x.Variables) + .Returns(new Variables(tc, copy: new Dictionary(), maskHints: new List(), warnings: out warnings)); + executionContext + .Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + trace.Info($"{tag}{message}"); + }); + executionContext + .Setup(x => x.WriteDebug) + .Returns(true); + executionContext.Object.Variables.Set(Constants.Variables.Build.SourceFolder, sourceFolder); + executionContext.Object.Variables.Set(Constants.Variables.Build.SourceBranch, sourceBranch); + executionContext.Object.Variables.Set(Constants.Variables.Build.SourceVersion, sourceVersion); + executionContext.Object.Variables.Set(Constants.Variables.System.EnableAccessToken, enableAuth.ToString()); + + return executionContext; + } + + private ServiceEndpoint GetTestSourceEndpoint(string url, bool clean, bool checkoutSubmodules) + { + var endpoint = new ServiceEndpoint(); + endpoint.Data[WellKnownEndpointData.Clean] = clean.ToString(); + endpoint.Data[WellKnownEndpointData.CheckoutSubmodules] = checkoutSubmodules.ToString(); + endpoint.Url = new Uri(url); + endpoint.Authorization = new EndpointAuthorization() + { + Scheme = EndpointAuthorizationSchemes.UsernamePassword + }; + endpoint.Authorization.Parameters[EndpointAuthorizationParameters.Username] = "someuser"; + endpoint.Authorization.Parameters[EndpointAuthorizationParameters.Password] = "SomePassword!"; + + return endpoint; + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceGitClone() + { + using (TestHostContext tc = new TestHostContext(this)) + { + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "master", "a596e13f5db8869f44574be0392fb8fe1e790ce4", false); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", false, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitClone(executionContext.Object, dumySourceFolder, It.Is(u => u.AbsoluteUri.Equals("https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, "a596e13f5db8869f44574be0392fb8fe1e790ce4", It.IsAny())); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceGitFetch() + { + using (TestHostContext tc = new TestHostContext(this)) + { + var trace = tc.GetTrace(); + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + try + { + Directory.CreateDirectory(dumySourceFolder); + string dumyGitFolder = Path.Combine(dumySourceFolder, ".git"); + Directory.CreateDirectory(dumyGitFolder); + string dumyGitConfig = Path.Combine(dumyGitFolder, "config"); + File.WriteAllText(dumyGitConfig, "test git confg file"); + + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "master", "a596e13f5db8869f44574be0392fb8fe1e790ce4", false); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", false, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitDisableAutoGC(executionContext.Object, dumySourceFolder)); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitFetch(executionContext.Object, dumySourceFolder, "origin", It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, "a596e13f5db8869f44574be0392fb8fe1e790ce4", It.IsAny())); + } + finally + { + IOUtil.DeleteDirectory(dumySourceFolder, CancellationToken.None); + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceGitClonePR() + { + using (TestHostContext tc = new TestHostContext(this)) + { + var trace = tc.GetTrace(); + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "refs/pull/12345", "a596e13f5db8869f44574be0392fb8fe1e790ce4", false); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", false, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitClone(executionContext.Object, dumySourceFolder, It.Is(u => u.AbsoluteUri.Equals("https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitFetch(executionContext.Object, dumySourceFolder, "origin", new List() { "+refs/pull/12345:refs/remotes/pull/12345" }, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, It.Is(s => s.Equals("refs/remotes/pull/12345")), It.IsAny())); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceGitFetchPR() + { + using (TestHostContext tc = new TestHostContext(this)) + { + var trace = tc.GetTrace(); + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + try + { + Directory.CreateDirectory(dumySourceFolder); + string dumyGitFolder = Path.Combine(dumySourceFolder, ".git"); + Directory.CreateDirectory(dumyGitFolder); + string dumyGitConfig = Path.Combine(dumyGitFolder, "config"); + File.WriteAllText(dumyGitConfig, "test git confg file"); + + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "refs/pull/12345/merge", "a596e13f5db8869f44574be0392fb8fe1e790ce4", false); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", false, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitDisableAutoGC(executionContext.Object, dumySourceFolder)); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitFetch(executionContext.Object, dumySourceFolder, "origin", new List() { "+refs/pull/12345/merge:refs/remotes/pull/12345/merge" }, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, "refs/remotes/pull/12345/merge", It.IsAny())); + } + finally + { + IOUtil.DeleteDirectory(dumySourceFolder, CancellationToken.None); + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceReCloneOnUrlNotMatch() + { + using (TestHostContext tc = new TestHostContext(this)) + { + var trace = tc.GetTrace(); + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + try + { + Directory.CreateDirectory(dumySourceFolder); + string dumyGitFolder = Path.Combine(dumySourceFolder, ".git"); + Directory.CreateDirectory(dumyGitFolder); + string dumyGitConfig = Path.Combine(dumyGitFolder, "config"); + File.WriteAllText(dumyGitConfig, "test git confg file"); + + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "refs/heads/users/user1", "", true); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", false, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + _gitCommandManager + .Setup(x => x.GitGetFetchUrl(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(new Uri("https://github.com/Microsoft/vsts-another-agent"))); + + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitClone(executionContext.Object, dumySourceFolder, It.Is(u => u.AbsoluteUri.Equals("https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, "refs/remotes/origin/users/user1", It.IsAny())); + } + finally + { + IOUtil.DeleteDirectory(dumySourceFolder, CancellationToken.None); + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetSourceGitFetchWithClean() + { + using (TestHostContext tc = new TestHostContext(this)) + { + var trace = tc.GetTrace(); + // Arrange. + string dumySourceFolder = Path.Combine(IOUtil.GetBinPath(), "SourceProviderL0"); + try + { + Directory.CreateDirectory(dumySourceFolder); + string dumyGitFolder = Path.Combine(dumySourceFolder, ".git"); + Directory.CreateDirectory(dumyGitFolder); + string dumyGitConfig = Path.Combine(dumyGitFolder, "config"); + File.WriteAllText(dumyGitConfig, "test git confg file"); + + var executionContext = GetTestExecutionContext(tc, dumySourceFolder, "refs/remotes/origin/master", "", false); + var endpoint = GetTestSourceEndpoint("https://github.com/Microsoft/vsts-agent", true, false); + + var _gitCommandManager = GetDefaultGitCommandMock(); + tc.SetSingleton(_gitCommandManager.Object); + tc.SetSingleton(new WhichUtil()); + + GitSourceProvider gitSourceProvider = new GitSourceProvider(); + gitSourceProvider.Initialize(tc); + + // Act. + gitSourceProvider.GetSourceAsync(executionContext.Object, endpoint, default(CancellationToken)).GetAwaiter().GetResult(); + + // Assert. + _gitCommandManager.Verify(x => x.GitClean(executionContext.Object, dumySourceFolder)); + _gitCommandManager.Verify(x => x.GitReset(executionContext.Object, dumySourceFolder)); + _gitCommandManager.Verify(x => x.GitDisableAutoGC(executionContext.Object, dumySourceFolder)); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", It.Is(s => s.Equals("https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")))); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", It.Is(s => s.Equals("https://someuser:SomePassword%21@github.com/Microsoft/vsts-agent")))); + _gitCommandManager.Verify(x => x.GitFetch(executionContext.Object, dumySourceFolder, "origin", It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + _gitCommandManager.Verify(x => x.GitRemoteSetUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitRemoteSetPushUrl(executionContext.Object, dumySourceFolder, "origin", "https://github.com/Microsoft/vsts-agent")); + _gitCommandManager.Verify(x => x.GitCheckout(executionContext.Object, dumySourceFolder, "refs/remotes/origin/master", It.IsAny())); + } + finally + { + IOUtil.DeleteDirectory(dumySourceFolder, CancellationToken.None); + } + } + } + } +}