diff --git a/src/Agent.Listener/Agent.Listener.csproj b/src/Agent.Listener/Agent.Listener.csproj index 82c1c97ab5..da3e81e56a 100644 --- a/src/Agent.Listener/Agent.Listener.csproj +++ b/src/Agent.Listener/Agent.Listener.csproj @@ -18,9 +18,9 @@ - + - + diff --git a/src/Agent.Plugins/Artifact/ArtifactItemFilters.cs b/src/Agent.Plugins/Artifact/ArtifactItemFilters.cs index eacac7aaa2..97cd730431 100644 --- a/src/Agent.Plugins/Artifact/ArtifactItemFilters.cs +++ b/src/Agent.Plugins/Artifact/ArtifactItemFilters.cs @@ -11,6 +11,7 @@ using Agent.Sdk; using Microsoft.TeamFoundation.Build.WebApi; using Microsoft.VisualStudio.Services.BlobStore.Common; +using Microsoft.VisualStudio.Services.BlobStore.WebApi; using Microsoft.VisualStudio.Services.Content.Common.Tracing; using Microsoft.VisualStudio.Services.FileContainer; using Microsoft.VisualStudio.Services.WebApi; diff --git a/src/Agent.Plugins/Artifact/FileContainerProvider.cs b/src/Agent.Plugins/Artifact/FileContainerProvider.cs index 81cf0e1728..dd4f31c87b 100644 --- a/src/Agent.Plugins/Artifact/FileContainerProvider.cs +++ b/src/Agent.Plugins/Artifact/FileContainerProvider.cs @@ -156,7 +156,12 @@ private async Task DownloadFileContainerAsync(IEnumerable ite try { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( - false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); + false, + (str) => this.tracer.Info(str), + this.connection, + DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), + Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.BuildArtifact, + cancellationToken); } catch (SocketException e) { diff --git a/src/Agent.Sdk/Agent.Sdk.csproj b/src/Agent.Sdk/Agent.Sdk.csproj index 950d9bbcd5..343a52e590 100644 --- a/src/Agent.Sdk/Agent.Sdk.csproj +++ b/src/Agent.Sdk/Agent.Sdk.csproj @@ -8,8 +8,9 @@ - - + + + diff --git a/src/Agent.Worker/Agent.Worker.csproj b/src/Agent.Worker/Agent.Worker.csproj index 3328d9276a..85e8285a81 100644 --- a/src/Agent.Worker/Agent.Worker.csproj +++ b/src/Agent.Worker/Agent.Worker.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Agent.Worker/Build/FileContainerServer.cs b/src/Agent.Worker/Build/FileContainerServer.cs index 0cc220fc24..353862d13e 100644 --- a/src/Agent.Worker/Build/FileContainerServer.cs +++ b/src/Agent.Worker/Build/FileContainerServer.cs @@ -345,7 +345,13 @@ private async Task BlobUploadAsync(IAsyncCommandContext context, I var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); int maxParallelism = context.GetHostContext().GetService().GetSettings().MaxDedupParallelism; (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance - .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token); + .CreateDedupClientAsync( + verbose, + (str) => context.Output(str), + this._connection, + maxParallelism, + BlobStore.WebApi.Contracts.Client.BuildArtifact, + token); // Upload to blobstore var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) => diff --git a/src/Common.props b/src/Common.props index 789d317a5b..edb0e67ed3 100644 --- a/src/Common.props +++ b/src/Common.props @@ -10,7 +10,7 @@ OS_UNKNOWN ARCH_UNKNOWN - 0.5.222-private + 0.5.227-262a3469 $(CodeAnalysis) false diff --git a/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs b/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs index cf7cfde0c4..ba7c09215f 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs @@ -15,6 +15,7 @@ using BuildXL.Cache.ContentStore.Hashing; using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts; using Agent.Sdk.Knob; +using System.Collections.Generic; namespace Microsoft.VisualStudio.Services.Agent.Blob { @@ -27,7 +28,7 @@ public interface IDedupManifestArtifactClientFactory /// If true emit verbose telemetry. /// Action used for logging. /// VssConnection - /// Maximum number of parallel threads that should be used for download. If 0 then + /// Maximum number of parallel threads that should be used for download. If 0 then /// use the system default. /// Cancellation token used for both creating clients and verifying client conneciton. /// Tuple of the client and the telemtery client @@ -40,7 +41,7 @@ public interface IDedupManifestArtifactClientFactory ClientSettingsInfo clientSettings, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken); - + /// /// Creates a DedupManifestArtifactClient client and retrieves any client settings from the server /// @@ -60,8 +61,9 @@ public interface IDedupManifestArtifactClientFactory /// If true emit verbose telemetry. /// Action used for logging. /// VssConnection - /// Maximum number of parallel threads that should be used for download. If 0 then + /// Maximum number of parallel threads that should be used for download. If 0 then /// use the system default. + /// The client type for client settings /// Cancellation token used for both creating clients and verifying client conneciton. /// Tuple of the client and the telemtery client Task<(DedupStoreClient client, BlobStoreClientTelemetryTfs telemetry)> CreateDedupClientAsync( @@ -69,6 +71,7 @@ public interface IDedupManifestArtifactClientFactory Action traceOutput, VssConnection connection, int maxParallelism, + BlobStore.WebApi.Contracts.Client? clientType, CancellationToken cancellationToken); /// @@ -84,7 +87,7 @@ public class DedupManifestArtifactClientFactory : IDedupManifestArtifactClientFa // NOTE: this should be set to ClientSettingsConstants.DefaultDomainId when the latest update from Azure Devops is added. private static string DefaultDomainIdKey = "DefaultDomainId"; - // Old default for hosted agents was 16*2 cores = 32. + // Old default for hosted agents was 16*2 cores = 32. // In my tests of a node_modules folder, this 32x parallelism was consistently around 47 seconds. // At 192x it was around 16 seconds and 256x was no faster. private const int DefaultDedupStoreClientMaxParallelism = 192; @@ -115,7 +118,7 @@ private DedupManifestArtifactClientFactory() client, CreateArtifactsTracer(verbose, traceOutput), cancellationToken); - + return CreateDedupManifestClient( context.IsSystemDebugTrue(), (str) => context.Output(str), @@ -124,7 +127,7 @@ private DedupManifestArtifactClientFactory() domainId, clientSettings, context, - cancellationToken); + cancellationToken); } public (DedupManifestArtifactClient client, BlobStoreClientTelemetry telemetry) CreateDedupManifestClient( @@ -153,13 +156,13 @@ private DedupManifestArtifactClientFactory() tracer, cancellationToken); - var helper = new HttpRetryHelper(maxRetries,e => true); + var helper = new HttpRetryHelper(maxRetries, e => true); IDedupStoreHttpClient dedupStoreHttpClient = helper.Invoke( () => { // since our call below is hidden, check if we are cancelled and throw if we are... - cancellationToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); IDedupStoreHttpClient dedupHttpclient; // this is actually a hidden network call to the location service: @@ -185,7 +188,9 @@ private DedupManifestArtifactClientFactory() } traceOutput($"Hashtype: {this.HashType.Value}"); - var dedupClient = new DedupStoreClientWithDataport(dedupStoreHttpClient, new DedupStoreClientContext(maxParallelism), this.HashType.Value); + dedupStoreHttpClient.SetRedirectTimeout(GetRedirectTimeoutFromClientSettings(clientSettings)); + + var dedupClient = new DedupStoreClientWithDataport(dedupStoreHttpClient, new DedupStoreClientContext(maxParallelism), this.HashType.Value); return (new DedupManifestArtifactClient(telemetry, dedupClient, tracer), telemetry); } @@ -194,6 +199,7 @@ private DedupManifestArtifactClientFactory() Action traceOutput, VssConnection connection, int maxParallelism, + BlobStore.WebApi.Contracts.Client? clientType, CancellationToken cancellationToken) { const int maxRetries = 5; @@ -203,6 +209,15 @@ private DedupManifestArtifactClientFactory() maxParallelism = DefaultDedupStoreClientMaxParallelism; } traceOutput($"Max dedup parallelism: {maxParallelism}"); + + ClientSettingsInfo clientSettings = clientType is null + ? null + : await GetClientSettingsAsync( + connection, + clientType.Value, + tracer, + cancellationToken); + var dedupStoreHttpClient = await AsyncHttpRetryHelper.InvokeAsync( () => { @@ -222,6 +237,8 @@ private DedupManifestArtifactClientFactory() cancellationToken: cancellationToken, continueOnCapturedContext: false); + dedupStoreHttpClient.SetRedirectTimeout(GetRedirectTimeoutFromClientSettings(clientSettings)); + var telemetry = new BlobStoreClientTelemetryTfs(tracer, dedupStoreHttpClient.BaseAddress, connection); var client = new DedupStoreClient(dedupStoreHttpClient, maxParallelism); return (client, telemetry); @@ -305,7 +322,7 @@ public static async Task GetClientSettingsAsync( var blobUri = connection.GetClient().BaseAddress; var clientSettingsHttpClient = factory.CreateVssHttpClient(blobUri); - return await clientSettingsHttpClient.GetSettingsAsync(client, userState: null, cancellationToken); + return await clientSettingsHttpClient.GetSettingsAsync(client, userState: null, cancellationToken); } catch (Exception exception) { @@ -329,7 +346,7 @@ public static IDomainId GetDefaultDomainId(ClientSettingsInfo clientSettings, IA tracer.Info($"Error converting the domain id '{clientSettings.Properties[DefaultDomainIdKey]}': {exception.Message}. Falling back to default."); } } - + return domainId; } @@ -355,5 +372,17 @@ private static HashType GetClientHashType(ClientSettingsInfo clientSettings, Age return ChunkerHelper.IsHashTypeChunk(hashType) ? hashType : ChunkerHelper.DefaultChunkHashType; } - } + + private int? GetRedirectTimeoutFromClientSettings(ClientSettingsInfo clientSettings) + { + if (int.TryParse(clientSettings?.Properties.GetValueOrDefault(ClientSettingsConstants.RedirectTimeout), out int redirectTimeoutSeconds)) + { + return redirectTimeoutSeconds; + } + else + { + return null; + } + } + } } \ No newline at end of file diff --git a/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs b/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs index dddb220ae6..6c359a71ac 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs @@ -160,7 +160,7 @@ public async Task UploadLogToBlobStore(Stream blob, st BlobIdentifier blobId = VsoHash.CalculateBlobIdentifierWithBlocks(blob).BlobId; - // Since we read this while calculating the hash, the position needs to be reset before we send this + // Since we read this while calculating the hash, the position needs to be reset before we send this blob.Position = 0; using (var blobClient = CreateArtifactsClient(_connection, default(CancellationToken))) @@ -173,7 +173,13 @@ public async Task UploadLogToBlobStore(Stream blob, st { int maxParallelism = HostContext.GetService().GetSettings().MaxDedupParallelism; var (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance - .CreateDedupClientAsync(verbose, (str) => Trace.Info(str), this._connection, maxParallelism, cancellationToken); + .CreateDedupClientAsync( + verbose, + (str) => Trace.Info(str), + this._connection, + maxParallelism, + clientType: null, + cancellationToken); var results = await BlobStoreUtils.UploadToBlobStore(verbose, itemPath, (level, uri, type) => new TimelineRecordAttachmentTelemetryRecord(level, uri, type, nameof(UploadAttachmentToBlobStore), planId, jobId, Guid.Empty), (str) => Trace.Info(str), dedupClient, clientTelemetry, cancellationToken); diff --git a/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj b/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj index 30103259b4..42c73a7cc9 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj +++ b/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs b/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs index e00124ed09..0380b35fb5 100644 --- a/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs +++ b/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs @@ -54,7 +54,13 @@ public class MockDedupManifestArtifactClientFactory : IDedupManifestArtifactClie telemetrySender)); } - public Task<(DedupStoreClient client, BlobStoreClientTelemetryTfs telemetry)> CreateDedupClientAsync(bool verbose, Action traceOutput, VssConnection connection, int maxParallelism, CancellationToken cancellationToken) + public Task<(DedupStoreClient client, BlobStoreClientTelemetryTfs telemetry)> CreateDedupClientAsync( + bool verbose, + Action traceOutput, + VssConnection connection, + int maxParallelism, + BlobStore.WebApi.Contracts.Client? clientType, + CancellationToken cancellationToken) { telemetrySender = new TestTelemetrySender(); return Task.FromResult((client: (DedupStoreClient)null, telemetry: new BlobStoreClientTelemetryTfs(