From 6ded01710ff4afa7b9ffe86db0edba374ef9b2a2 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 11:00:17 -0700 Subject: [PATCH 01/10] Add WorkerProxy one-worker assignment and readiness APIs Add immutable assignment state, revision polling, relay lifecycle readiness, and injectable capability finalization. Expose management-only readiness, assignment, and instance-state endpoints with Host-aligned validation errors and source-generated JSON serialization. Includes state, relay, serialization, and endpoint integration coverage for #11974. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Http/WorkerHttpCapabilityProvider.cs | 2 + .../Management/InstanceStatePollRequest.cs | 12 + .../Management/ManagementApiEndpoints.cs | 66 ++ .../Management/ManagementApiHandlers.cs | 136 ++++ .../Management/RequestValidationError.cs | 9 + .../Management/RequestValidationResponse.cs | 11 + .../Management/WorkerApiError.cs | 17 + .../Management/WorkerApiErrorCodes.cs | 20 + .../Management/WorkerApiErrorResponse.cs | 9 + .../Management/WorkerAssignRequest.cs | 22 + .../Management/WorkerInstanceState.cs | 20 + .../Management/WorkerPodStateResponse.cs | 15 + .../Management/WorkerProxyJsonContext.cs | 17 + .../Rpc/FunctionRpcRelay.Session.cs | 67 +- .../Rpc/FunctionRpcRelay.cs | 12 +- .../Rpc/IWorkerCapabilityFinalizer.cs | 25 + .../State/WorkerAssignment.cs | 88 +++ .../State/WorkerAssignmentResult.cs | 15 + .../State/WorkerAssignmentState.cs | 16 + .../State/WorkerPodState.cs | 42 ++ .../State/WorkerPodStateManager.cs | 294 +++++++++ .../State/WorkerPodStatus.cs | 13 + .../State/WorkerStatePollResult.cs | 17 + .../WorkerProxyApplication.cs | 9 +- .../WorkerProxyOptions.cs | 7 +- .../WorkerProxyOptionsValidator.cs | 7 +- ...orkerEndpointReadinessProbeOptionsTests.cs | 2 +- .../Management/ManagementApiEndpointTests.cs | 293 +++++++++ .../Management/ManagementApiHandlersTests.cs | 605 ++++++++++++++++++ .../Management/WorkerProxyJsonContextTests.cs | 307 +++++++++ ...ionRpcRelayTests.CapabilityFinalization.cs | 168 +++++ .../FunctionRpcRelayTests.HttpCapabilities.cs | 45 +- .../Rpc/FunctionRpcRelayTests.Management.cs | 229 +++++++ .../Rpc/FunctionRpcRelayTests.Readiness.cs | 338 ++++++++++ .../Rpc/FunctionRpcRelayTests.cs | 34 +- .../State/WorkerAssignmentTests.cs | 98 +++ .../WorkerPodStateManagerTests.Polling.cs | 375 +++++++++++ .../State/WorkerPodStateManagerTests.cs | 340 ++++++++++ .../WorkerProxyApplicationTests.cs | 73 ++- ...orkerProxyEnvironmentConfigurationTests.cs | 3 + .../WorkerProxyOptionsTests.cs | 4 +- .../WorkerProxyWebApplicationFactory.cs | 1 + 42 files changed, 3845 insertions(+), 38 deletions(-) create mode 100644 src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs create mode 100644 src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs create mode 100644 src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs create mode 100644 src/Functions.WorkerProxy/Management/RequestValidationError.cs create mode 100644 src/Functions.WorkerProxy/Management/RequestValidationResponse.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerApiError.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerInstanceState.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs create mode 100644 src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs create mode 100644 src/Functions.WorkerProxy/Rpc/IWorkerCapabilityFinalizer.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerAssignment.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerAssignmentState.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerPodState.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerPodStateManager.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerPodStatus.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerStatePollResult.cs create mode 100644 test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs create mode 100644 test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs create mode 100644 test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs create mode 100644 test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs create mode 100644 test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs create mode 100644 test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs create mode 100644 test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs create mode 100644 test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs create mode 100644 test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs diff --git a/src/Functions.WorkerProxy/Http/WorkerHttpCapabilityProvider.cs b/src/Functions.WorkerProxy/Http/WorkerHttpCapabilityProvider.cs index ffa8b70789..ed40d15937 100644 --- a/src/Functions.WorkerProxy/Http/WorkerHttpCapabilityProvider.cs +++ b/src/Functions.WorkerProxy/Http/WorkerHttpCapabilityProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Azure.Functions.WorkerProxy.Rpc; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -12,6 +13,7 @@ namespace Azure.Functions.WorkerProxy.Http; /// Captures the worker's HTTP destination before advertising WorkerProxy to the runtime. /// internal sealed partial class WorkerHttpCapabilityProvider(IOptions options, ILogger logger) + : IWorkerCapabilityFinalizer { private const string HttpUriCapability = "HttpUri"; diff --git a/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs b/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs new file mode 100644 index 0000000000..adb0fea557 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Requests a current snapshot when the revision is omitted or null, or waits for a newer revision when supplied. +/// +internal sealed class InstanceStatePollRequest +{ + public long? LastKnownRevision { get; init; } +} diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs new file mode 100644 index 0000000000..5a67d6f222 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -0,0 +1,66 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.State; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Registers worker lifecycle APIs on the management listener. +/// +internal static class ManagementApiEndpoints +{ + public static void Map(IEndpointRouteBuilder endpoints) + { + endpoints.MapGet("/admin/worker/ready", ManagementApiHandlers.GetWorkerReady).AllowAnonymous(); + endpoints.MapPost("/admin/worker/assign", AssignWorkerAsync).AllowAnonymous(); + endpoints.MapPost("/admin/infra/instanceState", GetInstanceStateAsync).AllowAnonymous(); + } + + private static async Task AssignWorkerAsync(HttpRequest request, WorkerPodStateManager manager) + { + if (!request.HasJsonContentType()) + { + return ManagementApiHandlers.InvalidBody(); + } + + WorkerAssignRequest? assignment; + try + { + assignment = await request.ReadFromJsonAsync( + WorkerProxyJsonContext.Default.WorkerAssignRequest, request.HttpContext.RequestAborted); + } + catch (JsonException) + { + return ManagementApiHandlers.InvalidBody(); + } + + return ManagementApiHandlers.AssignWorker(assignment, manager); + } + + private static async Task GetInstanceStateAsync(HttpRequest request, WorkerPodStateManager manager) + { + if (!request.HasJsonContentType()) + { + return ManagementApiHandlers.InvalidBody(); + } + + InstanceStatePollRequest? poll; + try + { + poll = await request.ReadFromJsonAsync( + WorkerProxyJsonContext.Default.InstanceStatePollRequest, request.HttpContext.RequestAborted); + } + catch (JsonException) + { + return ManagementApiHandlers.InvalidBody(); + } + + return await ManagementApiHandlers.GetInstanceStateAsync(poll, manager, request.HttpContext.RequestAborted); + } +} diff --git a/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs new file mode 100644 index 0000000000..869f8db7a0 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs @@ -0,0 +1,136 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.State; +using Microsoft.AspNetCore.Http; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Maps validated management requests and worker lifecycle outcomes to HTTP results. +/// +internal static class ManagementApiHandlers +{ + public static IResult GetWorkerReady(WorkerPodStateManager manager) => + manager.State.IsWorkerReady ? TypedResults.Ok() : TypedResults.StatusCode(StatusCodes.Status503ServiceUnavailable); + + public static IResult AssignWorker(WorkerAssignRequest? request, WorkerPodStateManager manager) + { + if (request is null) + { + return InvalidBody(); + } + + List errors = []; + if (string.IsNullOrWhiteSpace(request.FunctionAppName)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionAppName")); + } + + if (string.IsNullOrWhiteSpace(request.FunctionGroupName)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionGroupName")); + } + + if (request.IsAlwaysReady is null) + { + errors.Add(new(WorkerApiErrorCodes.Required, "isAlwaysReady")); + } + + if (string.IsNullOrWhiteSpace(request.FunctionAppDirectory)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionAppDirectory")); + } + + Dictionary environment = new(StringComparer.Ordinal); + if (request.Environment is null) + { + errors.Add(new(WorkerApiErrorCodes.Required, "environment")); + } + else + { + foreach ((string key, string? value) in request.Environment) + { + if (string.IsNullOrEmpty(key) || value is null) + { + // Report the invalid field once without exposing environment keys or values. + errors.Add(new(WorkerApiErrorCodes.InvalidValue, "environment")); + break; + } + + environment.Add(key, value); + } + } + + if (errors.Count > 0 + || request.FunctionAppName is not { } functionAppName + || request.FunctionGroupName is not { } functionGroupName + || request.FunctionAppDirectory is not { } functionAppDirectory + || request.IsAlwaysReady is not { } isAlwaysReady) + { + return ValidationError(errors); + } + + WorkerAssignment assignment = new( + functionAppName, + functionGroupName, + isAlwaysReady, + environment, + functionAppDirectory); + return manager.Assign(assignment) switch + { + WorkerAssignmentResult.Success => TypedResults.Ok(), + WorkerAssignmentResult.WorkerNotReady => Error( + StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerNotReady, "The worker has not established a valid StartStream."), + WorkerAssignmentResult.AssignmentConflict => Error( + StatusCodes.Status409Conflict, WorkerApiErrorCodes.AssignmentConflict, "The pod is already assigned to a different assignment."), + WorkerAssignmentResult.WorkerTerminated => Error( + StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerTerminated, "The assigned worker stream has terminated."), + _ => throw new InvalidOperationException("Unexpected worker assignment result.") + }; + } + + public static async Task GetInstanceStateAsync( + InstanceStatePollRequest? request, + WorkerPodStateManager manager, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (request is null) + { + return InvalidBody(); + } + + if (request.LastKnownRevision is not { } lastKnownRevision) + { + return StateResponse(manager.State); + } + + // Revisions never decrease, so a revision valid here remains valid when the manager registers the poll. + if (lastKnownRevision < 0 || lastKnownRevision > manager.State.Revision) + { + return ValidationError([new(WorkerApiErrorCodes.InvalidRevision, "lastKnownRevision")]); + } + + WorkerStatePollResult result = await manager.WaitForChangeAsync(lastKnownRevision, cancellationToken); + return result.State is { } state ? StateResponse(state) : TypedResults.NoContent(); + } + + internal static IResult InvalidBody() => + ValidationError([new(WorkerApiErrorCodes.InvalidBody, "request")]); + + private static IResult ValidationError(IReadOnlyList errors) => + TypedResults.Json(new RequestValidationResponse(errors), + WorkerProxyJsonContext.Default.RequestValidationResponse, statusCode: StatusCodes.Status400BadRequest); + + private static IResult Error(int statusCode, string code, string detail) => + TypedResults.Json(new WorkerApiErrorResponse(new(code, detail)), + WorkerProxyJsonContext.Default.WorkerApiErrorResponse, statusCode: statusCode); + + private static IResult StateResponse(WorkerPodState state) => + TypedResults.Json(WorkerInstanceState.FromState(state), WorkerProxyJsonContext.Default.WorkerInstanceState); +} diff --git a/src/Functions.WorkerProxy/Management/RequestValidationError.cs b/src/Functions.WorkerProxy/Management/RequestValidationError.cs new file mode 100644 index 0000000000..49454efe24 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/RequestValidationError.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Identifies an invalid request field, or request for a body-level error, without echoing its value. +/// +internal sealed record RequestValidationError(string Code, string Target); diff --git a/src/Functions.WorkerProxy/Management/RequestValidationResponse.cs b/src/Functions.WorkerProxy/Management/RequestValidationResponse.cs new file mode 100644 index 0000000000..a770b260e5 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/RequestValidationResponse.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Contains detected field errors, or one request-body error, returned with HTTP 400. +/// +internal sealed record RequestValidationResponse(IReadOnlyList Errors); diff --git a/src/Functions.WorkerProxy/Management/WorkerApiError.cs b/src/Functions.WorkerProxy/Management/WorkerApiError.cs new file mode 100644 index 0000000000..6b8dedcb41 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerApiError.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Provides a stable error code and optional diagnostic detail without echoing request values. +/// +/// A case-sensitive contract identifier; existing codes must not be renamed or repurposed. +/// Diagnostic text that may change and must not be used for client decisions. +/// +/// WorkerNotReady (503) permits retry after readiness. WorkerTerminated (503) is terminal for the +/// assigned session; retrying the same assignment on this pod cannot recover it. +/// AssignmentConflict (409) rejects a different assignment; do not retry that request unchanged. +/// Clients must inspect Code to distinguish the two 503 outcomes and handle unknown codes gracefully. +/// +internal sealed record WorkerApiError(string Code, string? Detail = null); diff --git a/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs b/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs new file mode 100644 index 0000000000..414c266413 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Defines the stable error codes returned by the management APIs. +/// +internal static class WorkerApiErrorCodes +{ + // Clients branch on these exact, case-sensitive wire values. Do not change or repurpose them. + // Keep explicit literals rather than nameof so symbol renames cannot change the contract. + public const string Required = "Required"; + public const string InvalidBody = "InvalidBody"; + public const string InvalidValue = "InvalidValue"; + public const string InvalidRevision = "InvalidRevision"; + public const string WorkerNotReady = "WorkerNotReady"; + public const string WorkerTerminated = "WorkerTerminated"; + public const string AssignmentConflict = "AssignmentConflict"; +} diff --git a/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs b/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs new file mode 100644 index 0000000000..c5429deab7 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Wraps validation and lifecycle failures in the same management API error envelope. +/// +internal sealed record WorkerApiErrorResponse(WorkerApiError Error); diff --git a/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs b/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs new file mode 100644 index 0000000000..af40cea706 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Describes assignment identity for an already-specialized worker. +/// +internal sealed class WorkerAssignRequest +{ + public string? FunctionAppName { get; init; } + + public string? FunctionGroupName { get; init; } + + public bool? IsAlwaysReady { get; init; } + + public Dictionary? Environment { get; init; } + + public string? FunctionAppDirectory { get; init; } +} diff --git a/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs b/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs new file mode 100644 index 0000000000..feace916af --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Azure.Functions.WorkerProxy.State; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Exposes platform-facing pod state without internal session details or assignment environment values. +/// +internal sealed record WorkerInstanceState( + string PodName, + long RevisionId, + WorkerPodStateResponse WorkerPodState) +{ + public string FunctionsContainerType => "FunctionsWorkerPod"; + + public static WorkerInstanceState FromState(WorkerPodState state) => + new(state.PodName, state.Revision, new(state.PodStatus, state.FunctionGroupName, state.IsAlwaysReady)); +} diff --git a/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs b/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs new file mode 100644 index 0000000000..c486ead7ee --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Text.Json.Serialization; +using Azure.Functions.WorkerProxy.State; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Reports worker-pod eligibility and known assignment identity, not runtime serving readiness. +/// +internal sealed record WorkerPodStateResponse( + [property: JsonConverter(typeof(JsonStringEnumConverter))] WorkerPodStatus PodStatus, + string? FunctionGroupName, + bool? IsAlwaysReady); diff --git a/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs b/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs new file mode 100644 index 0000000000..b22ee1d790 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Text.Json.Serialization; + +namespace Azure.Functions.WorkerProxy.Management; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(WorkerAssignRequest))] +[JsonSerializable(typeof(InstanceStatePollRequest))] +[JsonSerializable(typeof(WorkerInstanceState))] +[JsonSerializable(typeof(WorkerApiErrorResponse))] +[JsonSerializable(typeof(RequestValidationResponse))] +internal sealed partial class WorkerProxyJsonContext : JsonSerializerContext; diff --git a/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.Session.cs b/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.Session.cs index d6dc16bca9..9003ba557b 100644 --- a/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.Session.cs +++ b/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.Session.cs @@ -3,10 +3,11 @@ using System; using System.Collections.Frozen; +using System.IO; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; -using Azure.Functions.WorkerProxy.Http; +using Azure.Functions.WorkerProxy.State; using Grpc.Core; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using Microsoft.Extensions.Logging; @@ -18,7 +19,11 @@ internal sealed partial class FunctionRpcRelay /// /// Owns the queues, forwarding tasks, and terminal state for one runtime/worker stream pair. /// - private sealed class FunctionRpcRelaySession(long id, ILogger logger, WorkerHttpCapabilityProvider capabilityProvider) + private sealed class FunctionRpcRelaySession( + long id, + ILogger logger, + IWorkerCapabilityFinalizer capabilityFinalizer, + WorkerPodStateManager stateManager) { private readonly Lock _stateLock = new(); private readonly Channel _toRuntime = CreateChannel(); @@ -93,6 +98,12 @@ public FunctionRpcRelayAttachResult TryAttach(FunctionRpcRelaySide side) } SetAttachedLocked(side, value: true); + if (side == FunctionRpcRelaySide.Worker) + { + // Observe admission without changing relay replacement policy; a failed assignment stays terminal. + stateManager.OnWorkerAttached(id); + } + return FunctionRpcRelayAttachResult.Attached; } } @@ -169,6 +180,11 @@ public void Detach(FunctionRpcRelaySide side) } SetAttachedLocked(side, value: false); + if (side == FunctionRpcRelaySide.Worker) + { + stateManager.OnSessionTerminated(id); + } + SignalReleasedIfCompleteLocked(); } } @@ -194,22 +210,49 @@ private static Channel CreateChannel() private async Task ReadInboundAsync(FunctionRpcRelaySide side, IAsyncStreamReader requestStream, ChannelWriter destination, CancellationToken cancellationToken) { + bool isFirstMessage = true; while (await requestStream.MoveNext(cancellationToken)) { - StreamingMessage message = requestStream.Current; - if (side is FunctionRpcRelaySide.Worker - && message.WorkerInitResponse is { Result.Status: StatusResult.Types.Status.Success }) + StreamingMessage? message = ProcessInboundMessage(side, requestStream.Current, isFirstMessage); + if (message is null) + { + return; + } + + isFirstMessage = false; + await destination.WriteAsync(message, cancellationToken); + } + } + + private StreamingMessage? ProcessInboundMessage(FunctionRpcRelaySide side, StreamingMessage message, bool isFirstMessage) + { + if (side != FunctionRpcRelaySide.Worker) + { + return message; + } + + if (isFirstMessage) + { + lock (_stateLock) { - if (FinalizeCapabilities(message) is not { } finalized) + // A delayed read from a terminated session must not restore readiness. + if (_terminalState is not null) { - return; + return null; } - message = finalized; - } + if (message.StartStream is not { } startStream || string.IsNullOrWhiteSpace(startStream.WorkerId)) + { + throw new InvalidDataException("The first worker message must be StartStream with a nonempty worker ID."); + } - await destination.WriteAsync(message, cancellationToken); + stateManager.OnWorkerStartStream(id, startStream.WorkerId); + } } + + return message.WorkerInitResponse is { Result.Status: StatusResult.Types.Status.Success } + ? FinalizeCapabilities(message) + : message; } private StreamingMessage? FinalizeCapabilities(StreamingMessage message) @@ -229,7 +272,7 @@ private async Task ReadInboundAsync(FunctionRpcRelaySide side, IAsyncStreamReade StreamingMessage finalized = message.Clone(); if (capabilities is null) { - Uri? destination = capabilityProvider.FinalizeCapabilities(finalized.WorkerInitResponse.Capabilities); + Uri? destination = capabilityFinalizer.FinalizeCapabilities(finalized.WorkerInitResponse.Capabilities); capabilities = finalized.WorkerInitResponse.Capabilities.ToFrozenDictionary(StringComparer.Ordinal); lock (_stateLock) { @@ -306,6 +349,8 @@ private bool TryTerminate(FunctionRpcRelayTerminalState terminalState) } _terminalState = terminalState; + // Withdraw readiness before completing stream tasks or starting potentially slow teardown/logging. + stateManager.OnSessionTerminated(id); _completion.SetResult(terminalState); } diff --git a/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.cs b/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.cs index c79db3bce7..a70807a289 100644 --- a/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.cs +++ b/src/Functions.WorkerProxy/Rpc/FunctionRpcRelay.cs @@ -4,7 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; -using Azure.Functions.WorkerProxy.Http; +using Azure.Functions.WorkerProxy.State; using Grpc.Core; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using Microsoft.Extensions.Hosting; @@ -19,14 +19,17 @@ namespace Azure.Functions.WorkerProxy.Rpc; /// A relay session admits one stream per . The first peer close, /// cancellation, stream failure, or application shutdown terminates the whole session. A replacement /// session is created only after both attachments from the previous session have released. +/// Locks are acquired in relay, session, then pod-state order; the state manager never calls back into the relay. /// internal sealed partial class FunctionRpcRelay( ILogger logger, - WorkerHttpCapabilityProvider capabilityProvider) + IWorkerCapabilityFinalizer capabilityFinalizer, + WorkerPodStateManager stateManager) : IAsyncDisposable, IHostedLifecycleService { private readonly Lock _syncLock = new(); - private readonly WorkerHttpCapabilityProvider _capabilityProvider = capabilityProvider ?? throw new ArgumentNullException(nameof(capabilityProvider)); + private readonly IWorkerCapabilityFinalizer _capabilityFinalizer = capabilityFinalizer ?? throw new ArgumentNullException(nameof(capabilityFinalizer)); + private readonly WorkerPodStateManager _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); // Teardown continues independently of each caller's wait token; every StopAsync and DisposeAsync joins this completion. private readonly TaskCompletionSource _stopCompletionSource = new(TaskCreationOptions.RunContinuationsAsynchronously); private FunctionRpcRelaySession? _currentSession; @@ -135,7 +138,8 @@ public Task AttachAsync(FunctionRpcRelaySide side ClearCurrentSessionLocked(); } - session = _currentSession ??= new FunctionRpcRelaySession(Interlocked.Increment(ref _nextSessionId), logger, _capabilityProvider); + session = _currentSession ??= new FunctionRpcRelaySession( + Interlocked.Increment(ref _nextSessionId), logger, _capabilityFinalizer, _stateManager); FunctionRpcRelayAttachResult attachResult = session.TryAttach(side); if (attachResult != FunctionRpcRelayAttachResult.Attached) diff --git a/src/Functions.WorkerProxy/Rpc/IWorkerCapabilityFinalizer.cs b/src/Functions.WorkerProxy/Rpc/IWorkerCapabilityFinalizer.cs new file mode 100644 index 0000000000..052025a6bd --- /dev/null +++ b/src/Functions.WorkerProxy/Rpc/IWorkerCapabilityFinalizer.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace Azure.Functions.WorkerProxy.Rpc; + +/// +/// Finalizes the worker capabilities advertised to the runtime during normal initialization. +/// +internal interface IWorkerCapabilityFinalizer +{ + /// + /// Updates capabilities on a relay-owned response copy before the relay freezes and forwards them. + /// + /// The mutable capability map from the cloned successful initialization response. + /// The real worker HTTP destination, or when HTTP proxying is unavailable. + /// + /// Called once per session on its first successful worker initialization response, outside relay lifecycle locks. + /// Failures propagate to session termination rather than forwarding a partially finalized response. + /// Implementations must not retain or mutate the capability map after returning. + /// + Uri? FinalizeCapabilities(IDictionary capabilities); +} diff --git a/src/Functions.WorkerProxy/State/WorkerAssignment.cs b/src/Functions.WorkerProxy/State/WorkerAssignment.cs new file mode 100644 index 0000000000..f4710466d2 --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerAssignment.cs @@ -0,0 +1,88 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Frozen; +using System.Collections.Generic; + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Captures immutable assignment identity for an already-specialized worker. +/// +/// +/// Environment and app directory participate in retry equality only; this model does not apply them +/// to the worker. Keep environment values out of published pod snapshots and diagnostic output. +/// +internal sealed class WorkerAssignment +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// Copies environment entries using ordinal key equality. + /// + public WorkerAssignment( + string functionAppName, + string functionGroupName, + bool isAlwaysReady, + IReadOnlyDictionary environment, + string functionAppDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(functionAppName); + ArgumentException.ThrowIfNullOrWhiteSpace(functionGroupName); + ArgumentNullException.ThrowIfNull(environment); + ArgumentException.ThrowIfNullOrWhiteSpace(functionAppDirectory); + + foreach ((string key, string value) in environment) + { + ArgumentException.ThrowIfNullOrEmpty(key); + ArgumentNullException.ThrowIfNull(value); + } + + FunctionAppName = functionAppName; + FunctionGroupName = functionGroupName; + IsAlwaysReady = isAlwaysReady; + // Read-only input can still wrap a mutable dictionary. Freeze it so later caller edits cannot change retry identity. + Environment = environment.ToFrozenDictionary(StringComparer.Ordinal); + FunctionAppDirectory = functionAppDirectory; + } + + public string FunctionAppName { get; } + + public string FunctionGroupName { get; } + + public bool IsAlwaysReady { get; } + + public FrozenDictionary Environment { get; } + + public string FunctionAppDirectory { get; } + + /// + /// Compares all assignment fields without depending on dictionary order or the caller's comparer. + /// + public bool IsEquivalentTo(WorkerAssignment other) + { + ArgumentNullException.ThrowIfNull(other); + + if (!string.Equals(FunctionAppName, other.FunctionAppName, StringComparison.Ordinal) + || !string.Equals(FunctionGroupName, other.FunctionGroupName, StringComparison.Ordinal) + || IsAlwaysReady != other.IsAlwaysReady + || !string.Equals(FunctionAppDirectory, other.FunctionAppDirectory, StringComparison.Ordinal) + || Environment.Count != other.Environment.Count) + { + return false; + } + + foreach ((string key, string value) in Environment) + { + if (!other.Environment.TryGetValue(key, out string? otherValue) + || !string.Equals(value, otherValue, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } +} diff --git a/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs b/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs new file mode 100644 index 0000000000..e951cd6113 --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Describes an assignment outcome independently of its HTTP representation. +/// +internal enum WorkerAssignmentResult +{ + Success, + WorkerNotReady, + AssignmentConflict, + WorkerTerminated +} diff --git a/src/Functions.WorkerProxy/State/WorkerAssignmentState.cs b/src/Functions.WorkerProxy/State/WorkerAssignmentState.cs new file mode 100644 index 0000000000..070fa7fa2d --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerAssignmentState.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Describes the internal lifecycle of a single worker assignment. +/// +internal enum WorkerAssignmentState +{ + Unassigned, + // The manager publishes assignment and readiness atomically, without exposing this intermediate state. + Assigned, + Ready, + Failed +} diff --git a/src/Functions.WorkerProxy/State/WorkerPodState.cs b/src/Functions.WorkerProxy/State/WorkerPodState.cs new file mode 100644 index 0000000000..9f23b17954 --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerPodState.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Provides an immutable lifecycle snapshot, without assignment environment values or wire-format concerns. +/// +internal sealed record WorkerPodState( + string PodName, + long Revision, + long? SessionId, + bool IsWorkerAttached, + string? WorkerId, + WorkerAssignmentState AssignmentState, + string? FunctionAppName, + string? FunctionGroupName, + bool? IsAlwaysReady) +{ + // WorkerId is recorded only after validated StartStream. Retaining it after termination does not imply readiness. + public bool IsWorkerReady => IsWorkerAttached && WorkerId is not null; + + // This is worker-pod eligibility, not proof of runtime initialization, function loading, or HTTP serving readiness. + public WorkerPodStatus PodStatus => AssignmentState == WorkerAssignmentState.Ready && IsWorkerReady + ? WorkerPodStatus.ReadyForRequest + : WorkerPodStatus.None; + + /// + /// Creates an unassigned pod snapshot at revision zero with no worker attached. + /// + public static WorkerPodState CreateUnassigned(string podName) => + new( + PodName: podName, + Revision: 0, + SessionId: null, + IsWorkerAttached: false, + WorkerId: null, + AssignmentState: WorkerAssignmentState.Unassigned, + FunctionAppName: null, + FunctionGroupName: null, + IsAlwaysReady: null); +} diff --git a/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs new file mode 100644 index 0000000000..0420ce6e3a --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs @@ -0,0 +1,294 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Owns single-worker assignment and readiness for the lifetime of a WorkerProxy pod. +/// +/// +/// Lifecycle notifications use the relay's monotonically increasing session IDs. +/// Assignment is synchronous bookkeeping; it never initializes or specializes a worker. +/// All state decisions and snapshot replacements share one lock, including assignment racing with termination. +/// Readers can retain a snapshot after releasing the lock because subsequent transitions replace rather than mutate it. +/// +internal sealed class WorkerPodStateManager +{ + private static readonly TimeSpan PollTimeout = TimeSpan.FromSeconds(60); + private readonly Lock _stateLock = new(); + private readonly HashSet> _waiters = []; + private readonly TimeProvider _timeProvider; + private WorkerPodState _state; + private WorkerAssignment? _assignment; + // High-water mark for accepted or terminated sessions, including sessions that never had a worker. + private long _lastSessionId; + private int _assignmentClaimed; + + public WorkerPodStateManager( + IOptions options, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(timeProvider); + + string podName = options.Value.PodName; + ArgumentException.ThrowIfNullOrWhiteSpace(podName); + + _timeProvider = timeProvider; + _state = WorkerPodState.CreateUnassigned(podName); + } + + /// + /// Gets the current immutable state without incrementing its revision. + /// + public WorkerPodState State + { + get + { + lock (_stateLock) + { + return _state; + } + } + } + + internal int PendingWaiterCount + { + get + { + lock (_stateLock) + { + return _waiters.Count; + } + } + } + + /// + /// Returns a newer snapshot immediately, or waits up to 60 seconds for a state change. + /// + /// A changed snapshot, or on timeout. + /// + /// The last known revision is negative or greater than the current revision. + /// + /// The caller canceled the poll. + public async Task WaitForChangeAsync( + long lastKnownRevision, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(lastKnownRevision); + cancellationToken.ThrowIfCancellationRequested(); + + TaskCompletionSource waiter; + lock (_stateLock) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(lastKnownRevision, _state.Revision); + if (lastKnownRevision < _state.Revision) + { + return new WorkerStatePollResult(_state); + } + + // Compare and register under the publication lock so a transition cannot slip between them. + waiter = new(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add(waiter); + } + + try + { + WorkerPodState state = await waiter.Task.WaitAsync(PollTimeout, _timeProvider, cancellationToken).ConfigureAwait(false); + return new WorkerStatePollResult(state); + } + catch (TimeoutException) + { + // A transition may have won the publication lock just as the timer expired. + lock (_stateLock) + { + return _state.Revision > lastKnownRevision + ? new WorkerStatePollResult(_state) + : WorkerStatePollResult.NoChange; + } + } + finally + { + lock (_stateLock) + { + _waiters.Remove(waiter); + } + } + } + + /// + /// Records an accepted worker attachment. Returns false for old sessions or a terminal assignment. + /// + public bool OnWorkerAttached(long sessionId) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sessionId); + + lock (_stateLock) + { + // Pre-assignment transport sessions may be replaced; an assigned pod's failure is terminal. + if (sessionId <= _lastSessionId || _state.AssignmentState == WorkerAssignmentState.Failed) + { + return false; + } + + if (_state.IsWorkerAttached) + { + throw new InvalidOperationException("The current worker session must terminate before another worker attaches."); + } + + UpdateStateAndNotifyWaitersLocked(_state with + { + Revision = checked(_state.Revision + 1), + SessionId = sessionId, + IsWorkerAttached = true, + WorkerId = null + }); + _lastSessionId = sessionId; + return true; + } + } + + /// + /// Records the validated first StartStream. Returns false for stale, detached, or repeated notifications. + /// + /// + /// The relay must validate the first message's type before calling this method. + /// This manager validates the worker ID but does not parse FunctionRpc messages. + /// + public bool OnWorkerStartStream(long sessionId, string workerId) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sessionId); + ArgumentException.ThrowIfNullOrWhiteSpace(workerId); + + lock (_stateLock) + { + if (_state.SessionId != sessionId || !_state.IsWorkerAttached) + { + return false; + } + + if (_state.WorkerId is not null) + { + if (!string.Equals(_state.WorkerId, workerId, StringComparison.Ordinal)) + { + throw new InvalidOperationException("A worker session cannot change its StartStream identity."); + } + + return false; + } + + UpdateStateAndNotifyWaitersLocked(_state with { Revision = checked(_state.Revision + 1), WorkerId = workerId }); + return true; + } + } + + /// + /// Withdraws readiness on relay termination or worker detachment. Old notifications have no effect. + /// + public bool OnSessionTerminated(long sessionId) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sessionId); + + lock (_stateLock) + { + if (sessionId < _lastSessionId) + { + return false; + } + + if (_state.IsWorkerAttached && _state.SessionId != sessionId) + { + throw new InvalidOperationException("A different worker session is still attached."); + } + + // A runtime-only session can terminate before any worker attaches. Fence off its late notifications. + _lastSessionId = sessionId; + if (!_state.IsWorkerAttached) + { + return false; + } + + UpdateStateAndNotifyWaitersLocked(_state with + { + Revision = checked(_state.Revision + 1), + IsWorkerAttached = false, + // Preserve assignment identity so equivalent retries report failure and different assignments still conflict. + AssignmentState = _assignment is null ? WorkerAssignmentState.Unassigned : WorkerAssignmentState.Failed + }); + return true; + } + } + + /// + /// Records or replays assignment atomically with worker readiness and terminal state. + /// + public WorkerAssignmentResult Assign(WorkerAssignment assignment) + { + ArgumentNullException.ThrowIfNull(assignment); + + lock (_stateLock) + { + // Replay/conflict takes precedence over current readiness, including after the assigned stream terminates. + if (_assignment is not null) + { + if (!_assignment.IsEquivalentTo(assignment)) + { + return WorkerAssignmentResult.AssignmentConflict; + } + + return _state.AssignmentState == WorkerAssignmentState.Failed + ? WorkerAssignmentResult.WorkerTerminated + : WorkerAssignmentResult.Success; + } + + if (!_state.IsWorkerReady) + { + return WorkerAssignmentResult.WorkerNotReady; + } + + // No asynchronous specialization occurs: publish Ready directly, with a single revision increment. + WorkerPodState assigned = _state with + { + Revision = checked(_state.Revision + 1), + AssignmentState = WorkerAssignmentState.Ready, + FunctionAppName = assignment.FunctionAppName, + FunctionGroupName = assignment.FunctionGroupName, + IsAlwaysReady = assignment.IsAlwaysReady + }; + + // The gate and publication share the lifecycle lock, so a not-ready call never reserves identity. + if (Interlocked.CompareExchange(ref _assignmentClaimed, 1, 0) != 0) + { + throw new InvalidOperationException("Assignment identity was claimed without a recorded assignment."); + } + + _assignment = assignment; + UpdateStateAndNotifyWaitersLocked(assigned); + return WorkerAssignmentResult.Success; + } + } + + /// + /// Stores the new state and completes all pending polls with that snapshot, then clears the waiters. + /// + /// + /// The caller must hold . Waiting callers resume asynchronously, + /// rather than running inline when their tasks are completed. + /// + private void UpdateStateAndNotifyWaitersLocked(WorkerPodState state) + { + _state = state; + foreach (TaskCompletionSource waiter in _waiters) + { + waiter.TrySetResult(state); + } + + _waiters.Clear(); + } +} diff --git a/src/Functions.WorkerProxy/State/WorkerPodStatus.cs b/src/Functions.WorkerProxy/State/WorkerPodStatus.cs new file mode 100644 index 0000000000..9a5d4940df --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerPodStatus.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Describes the worker-pod status exposed to the platform. +/// +internal enum WorkerPodStatus +{ + None, + ReadyForRequest +} diff --git a/src/Functions.WorkerProxy/State/WorkerStatePollResult.cs b/src/Functions.WorkerProxy/State/WorkerStatePollResult.cs new file mode 100644 index 0000000000..54fe6a8e48 --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerStatePollResult.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Represents the outcome of waiting for a worker-pod state change. +/// +/// +/// State contains the updated snapshot when a change is observed, or is null when the wait times out without a change. +/// +internal sealed record WorkerStatePollResult(WorkerPodState? State) +{ + public static WorkerStatePollResult NoChange { get; } = new(State: null); + + public bool HasChanged => State is not null; +} diff --git a/src/Functions.WorkerProxy/WorkerProxyApplication.cs b/src/Functions.WorkerProxy/WorkerProxyApplication.cs index 14c3bb2d4a..9f954c29be 100644 --- a/src/Functions.WorkerProxy/WorkerProxyApplication.cs +++ b/src/Functions.WorkerProxy/WorkerProxyApplication.cs @@ -6,7 +6,9 @@ using System.Net.Http; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.Http; +using Azure.Functions.WorkerProxy.Management; using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; @@ -41,6 +43,10 @@ public static WebApplication Build(string[] args) builder.WebHost.UseSetting(WebHostDefaults.PreferHostingUrlsKey, bool.FalseString); builder.Services.AddOptions().BindConfiguration(WorkerProxyOptions.SectionName).ValidateOnStart(); builder.Services.AddSingleton, WorkerProxyOptionsValidator>(); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddSingleton(); + builder.Services.ConfigureHttpJsonOptions(options => + options.SerializerOptions.TypeInfoResolverChain.Insert(0, WorkerProxyJsonContext.Default)); builder.Services.AddSingleton(); builder.Services.AddSingleton>( static services => services.GetRequiredService()); @@ -69,6 +75,7 @@ private static void ConfigureManagementPipeline(IApplicationBuilder app) app.UseEndpoints(static endpoints => { endpoints.MapGet(ReadyPath, static () => Results.Ok()).AllowAnonymous(); + ManagementApiEndpoints.Map(endpoints); }); } @@ -119,7 +126,7 @@ private static void ConfigureHttpForwarding(WebApplicationBuilder builder) WorkerEndpointReadinessProbeOptionsValidator>(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddHttpForwarder(); builder.Services.AddHttpClient(nameof(WorkerHttpForwarder)) .ConfigurePrimaryHttpMessageHandler(static () => new SocketsHttpHandler diff --git a/src/Functions.WorkerProxy/WorkerProxyOptions.cs b/src/Functions.WorkerProxy/WorkerProxyOptions.cs index c120de0ae6..97a3e57d24 100644 --- a/src/Functions.WorkerProxy/WorkerProxyOptions.cs +++ b/src/Functions.WorkerProxy/WorkerProxyOptions.cs @@ -4,7 +4,7 @@ namespace Azure.Functions.WorkerProxy; /// -/// Defines the WorkerProxy listener ports. +/// Defines the WorkerProxy pod identity and listener settings. /// internal sealed class WorkerProxyOptions { @@ -13,6 +13,11 @@ internal sealed class WorkerProxyOptions /// public const string SectionName = "WorkerProxy"; + /// + /// Gets or sets the required platform-provided pod name. + /// + public string PodName { get; set; } = string.Empty; + /// /// Gets or sets the HTTP/1 management listener port. /// diff --git a/src/Functions.WorkerProxy/WorkerProxyOptionsValidator.cs b/src/Functions.WorkerProxy/WorkerProxyOptionsValidator.cs index b5be76bcc5..32e18db124 100644 --- a/src/Functions.WorkerProxy/WorkerProxyOptionsValidator.cs +++ b/src/Functions.WorkerProxy/WorkerProxyOptionsValidator.cs @@ -10,7 +10,7 @@ namespace Azure.Functions.WorkerProxy; /// -/// Validates WorkerProxy listener ports and the advertised HTTP origin. +/// Validates WorkerProxy pod identity, listener ports, and the advertised HTTP origin. /// internal sealed class WorkerProxyOptionsValidator : IValidateOptions { @@ -20,6 +20,11 @@ internal sealed class WorkerProxyOptionsValidator : IValidateOptions failures = []; + if (string.IsNullOrWhiteSpace(options.PodName)) + { + failures.Add($"{nameof(options.PodName)} must specify a nonempty platform-provided pod name."); + } + ValidatePort(options.ManagementPort, nameof(options.ManagementPort), failures); ValidatePort(options.RuntimeGrpcPort, nameof(options.RuntimeGrpcPort), failures); ValidatePort(options.WorkerGrpcPort, nameof(options.WorkerGrpcPort), failures); diff --git a/test/Functions.WorkerProxy.Tests/Http/WorkerEndpointReadinessProbeOptionsTests.cs b/test/Functions.WorkerProxy.Tests/Http/WorkerEndpointReadinessProbeOptionsTests.cs index d6a1a4ab34..2e014d9466 100644 --- a/test/Functions.WorkerProxy.Tests/Http/WorkerEndpointReadinessProbeOptionsTests.cs +++ b/test/Functions.WorkerProxy.Tests/Http/WorkerEndpointReadinessProbeOptionsTests.cs @@ -43,7 +43,7 @@ public void Options_RejectNonPositiveValues(string propertyName) private static WorkerEndpointReadinessProbeOptions GetOptions(params string[] args) { - using WebApplication app = WorkerProxyApplication.Build(args); + using WebApplication app = WorkerProxyApplication.Build(["--WorkerProxy:PodName", "test-worker-pod", .. args]); return app.Services.GetRequiredService>().Value; } diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs new file mode 100644 index 0000000000..da3eacc14f --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs @@ -0,0 +1,293 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Moq; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.Management; + +public class ManagementApiEndpointTests +{ + private const string AssignPath = "/admin/worker/assign"; + private const string StatePath = "/admin/infra/instanceState"; + private const string ValidAssignment = """ + {"functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, + "environment":{"SETTING":"private-value"},"functionAppDirectory":"/home/site/wwwroot"} + """; + + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(30); + + [Theory] + [InlineData("{}")] + [InlineData("{\"lastKnownRevision\":null}")] + public async Task InitialPollReturnsRevisionZeroImmediately(string body) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(body); + using HttpResponseMessage response = await client.PostAsync(StatePath, content, timeout.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + using JsonDocument state = JsonDocument.Parse(await response.Content.ReadAsStringAsync(timeout.Token)); + Assert.Equal(0, state.RootElement.GetProperty("revisionId").GetInt64()); + Assert.Equal("None", state.RootElement.GetProperty("workerPodState").GetProperty("podStatus").GetString()); + Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); + } + + [Theory] + [InlineData(AssignPath, "")] + [InlineData(AssignPath, "null")] + [InlineData(AssignPath, "[]")] + [InlineData(AssignPath, "{")] + [InlineData(StatePath, "")] + [InlineData(StatePath, "null")] + [InlineData(StatePath, "[]")] + [InlineData(StatePath, "{")] + [InlineData(StatePath, "{\"lastKnownRevision\":\"0\"}")] + [InlineData(StatePath, "{\"lastKnownRevision\":1.5}")] + [InlineData(StatePath, "{\"lastKnownRevision\":9223372036854775808}")] + public async Task MalformedBodyUsesHostValidationEnvelope(string path, string body) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(body); + using HttpResponseMessage response = await client.PostAsync(path, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); + Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); + } + + [Theory] + [InlineData("functionAppName", null, "Required")] + [InlineData("functionAppName", "\" \"", "Required")] + [InlineData("functionGroupName", null, "Required")] + [InlineData("functionGroupName", "\"\"", "Required")] + [InlineData("functionAppDirectory", null, "Required")] + [InlineData("functionAppDirectory", "\" \"", "Required")] + [InlineData("isAlwaysReady", null, "Required")] + [InlineData("isAlwaysReady", "null", "Required")] + [InlineData("isAlwaysReady", "\"false\"", "InvalidBody")] + [InlineData("environment", null, "Required")] + [InlineData("environment", "null", "Required")] + [InlineData("environment", "{\"\":\"private-value\"}", "InvalidValue")] + [InlineData("environment", "{\"SETTING\":null}", "InvalidValue")] + [InlineData("environment", "{\"SETTING\":123}", "InvalidBody")] + public async Task InvalidAssignmentDoesNotClaimIdentity(string field, string? value, string code) + { + JsonObject body = JsonNode.Parse(ValidAssignment)!.AsObject(); + if (value is null) + { + body.Remove(field); + } + else + { + body[field] = JsonNode.Parse(value); + } + + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(body.ToJsonString()); + using HttpResponseMessage response = await client.PostAsync(AssignPath, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, (code, code == "InvalidBody" ? "request" : field)); + WorkerPodState state = factory.Services.GetRequiredService().State; + Assert.Equal(0, state.Revision); + Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); + Assert.Null(state.FunctionAppName); + } + + [Theory] + [InlineData("{}")] + [InlineData(""" + {"functionAppName":" ","functionGroupName":"","isAlwaysReady":null, + "functionAppDirectory":" ","environment":{"PRIVATE_SETTING":null}} + """)] + public async Task AssignmentReturnsAllInvalidFieldsInOneResponse(string body) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(body); + using HttpResponseMessage response = await client.PostAsync(AssignPath, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, + ("Required", "functionAppName"), + ("Required", "functionGroupName"), + ("Required", "isAlwaysReady"), + ("Required", "functionAppDirectory"), + (body == "{}" ? "Required" : "InvalidValue", "environment")); + string json = await response.Content.ReadAsStringAsync(timeout.Token); + Assert.DoesNotContain("PRIVATE_SETTING", json); + Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); + } + + [Theory] + [InlineData(AssignPath)] + [InlineData(StatePath)] + public async Task NonJsonContentUsesHostValidationEnvelope(string path) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = new("{}", Encoding.UTF8, "text/plain"); + using HttpResponseMessage response = await client.PostAsync(path, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); + } + + [Theory] + [InlineData(-1)] + [InlineData(1)] + [InlineData(long.MaxValue)] + public async Task InvalidRevisionUsesHostValidationEnvelope(long revision) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody($"{{\"lastKnownRevision\":{revision}}}"); + using HttpResponseMessage response = await client.PostAsync(StatePath, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, ("InvalidRevision", "lastKnownRevision")); + Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); + } + + [Theory] + [InlineData("/admin/worker/ready", "GET")] + [InlineData(AssignPath, "POST")] + [InlineData(StatePath, "POST")] + public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string path, string method) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using CancellationTokenSource timeout = new(TestTimeout); + foreach (FunctionRpcRelaySide side in new[] { FunctionRpcRelaySide.Runtime, FunctionRpcRelaySide.Worker }) + { + using HttpClient rpcClient = new() { BaseAddress = factory.GetFunctionRpcAddress(side) }; + using HttpRequestMessage request = new(new HttpMethod(method), path) + { + Version = HttpVersion.Version20, + VersionPolicy = HttpVersionPolicy.RequestVersionExact, + Content = method == "POST" ? JsonBody("{}") : null + }; + using HttpResponseMessage response = await rpcClient.SendAsync(request, timeout.Token); + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + using HttpClient forwardingClient = factory.CreateHttpForwardingClient(); + using HttpRequestMessage forwardingRequest = new(new HttpMethod(method), path) + { + Content = method == "POST" ? JsonBody("{}") : null + }; + using HttpResponseMessage forwardingResponse = await forwardingClient.SendAsync(forwardingRequest, timeout.Token); + Assert.Equal(HttpStatusCode.NotFound, forwardingResponse.StatusCode); + } + + [Theory] + [InlineData("/admin/worker/ready", "POST")] + [InlineData(AssignPath, "GET")] + [InlineData(StatePath, "GET")] + public async Task ManagementRoutesRejectUnsupportedMethods(string path, string method) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using HttpRequestMessage request = new(new HttpMethod(method), path); + using HttpResponseMessage response = await client.SendAsync(request, timeout.Token); + + Assert.Equal(HttpStatusCode.MethodNotAllowed, response.StatusCode); + } + + [Fact] + public async Task EqualRevisionPollReturnsNoContentAtDeadline() + { + Mock clock = new(); + Mock timer = new(); + TaskCompletionSource expire = new(TaskCreationOptions.RunContinuationsAsynchronously); + clock.Setup(provider => provider.CreateTimer( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + { + Assert.Equal(TimeSpan.FromSeconds(60), dueTime); + Assert.Equal(Timeout.InfiniteTimeSpan, period); + expire.TrySetResult(() => callback(state)); + return timer.Object; + }); + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.Replace(ServiceDescriptor.Singleton(clock.Object))); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody("{\"lastKnownRevision\":0}"); + Task poll = client.PostAsync(StatePath, content, timeout.Token); + Action fireTimer = await expire.Task.WaitAsync(timeout.Token); + Assert.False(poll.IsCompleted); + + fireTimer(); + using HttpResponseMessage response = await poll; + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + Assert.Empty(await response.Content.ReadAsByteArrayAsync(timeout.Token)); + Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); + timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce); + } + + [Fact] + public async Task ClientCancellationRemovesPendingPoll() + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using CancellationTokenSource cancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); + using StringContent content = JsonBody("{\"lastKnownRevision\":0}"); + Task poll = client.PostAsync(StatePath, content, cancellation.Token); + while (manager.PendingWaiterCount == 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(10), timeout.Token); + } + + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => poll); + while (manager.PendingWaiterCount != 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(10), timeout.Token); + } + + Assert.Equal(0, manager.State.Revision); + } + + private static StringContent JsonBody(string body) => new(body, Encoding.UTF8, "application/json"); + + private static async Task AssertValidationAsync( + HttpResponseMessage response, CancellationToken cancellationToken, params (string Code, string Target)[] expected) + { + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); + string body = await response.Content.ReadAsStringAsync(cancellationToken); + using JsonDocument json = JsonDocument.Parse(body); + Assert.Equal("errors", Assert.Single(json.RootElement.EnumerateObject()).Name); + JsonElement errors = json.RootElement.GetProperty("errors"); + Assert.Equal(expected.Length, errors.GetArrayLength()); + for (int index = 0; index < expected.Length; index++) + { + Assert.Equal(new[] { "code", "target" }, errors[index].EnumerateObject().Select(property => property.Name).Order()); + Assert.Equal(expected[index].Code, errors[index].GetProperty("code").GetString()); + Assert.Equal(expected[index].Target, errors[index].GetProperty("target").GetString()); + } + + Assert.DoesNotContain("private-value", body); + } +} diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs new file mode 100644 index 0000000000..41189f0031 --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs @@ -0,0 +1,605 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Management; +using Azure.Functions.WorkerProxy.State; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.Management; + +public class ManagementApiHandlersTests +{ + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10); + + [Fact] + public void GetWorkerReady_TracksStartStreamAndTerminationWithoutChangingState() + { + WorkerPodStateManager manager = CreateManager(); + WorkerPodState initial = manager.State; + + Assert.Equal(503, Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)).StatusCode); + Assert.Same(initial, manager.State); + manager.OnWorkerAttached(1); + Assert.Equal(503, Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)).StatusCode); + + manager.OnWorkerStartStream(1, "worker"); + WorkerPodState started = manager.State; + Assert.Equal(200, Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)).StatusCode); + Assert.Same(started, manager.State); + Assert.Equal(WorkerPodStatus.None, started.PodStatus); + + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)); + manager.OnSessionTerminated(1); + WorkerPodState terminated = manager.State; + Assert.Equal(503, Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)).StatusCode); + Assert.Same(terminated, manager.State); + } + + [Fact] + public void AssignWorker_NullRequestIsInvalidEvenBeforeWorkerReady() + { + WorkerPodStateManager manager = CreateManager(); + WorkerPodState initial = manager.State; + + Assert.Equal(new("InvalidBody", "request"), Assert.Single( + AssertValidation(ManagementApiHandlers.AssignWorker(null, manager)))); + + Assert.Same(initial, manager.State); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AssignWorker_ReturnsAllMissingFieldsWithoutChangingState(bool ready) + { + WorkerPodStateManager manager = ready ? CreateReadyManager() : CreateManager(); + WorkerPodState before = manager.State; + + IReadOnlyList errors = AssertValidation(ManagementApiHandlers.AssignWorker(new(), manager)); + + Assert.Equal( + [ + new("Required", "functionAppName"), + new("Required", "functionGroupName"), + new("Required", "isAlwaysReady"), + new("Required", "functionAppDirectory"), + new("Required", "environment") + ], + errors); + Assert.Same(before, manager.State); + } + + [Fact] + public void AssignWorker_ReturnsAllBlankFieldsAndInvalidEnvironmentTogether() + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest request = new() + { + FunctionAppName = " ", + FunctionGroupName = string.Empty, + FunctionAppDirectory = "\t", + Environment = new() { [string.Empty] = "private-value", ["PRIVATE_SETTING"] = null } + }; + + IReadOnlyList errors = AssertValidation(ManagementApiHandlers.AssignWorker(request, manager)); + + Assert.Equal( + [ + new("Required", "functionAppName"), + new("Required", "functionGroupName"), + new("Required", "isAlwaysReady"), + new("Required", "functionAppDirectory"), + new("InvalidValue", "environment") + ], + errors); + Assert.Same(before, manager.State); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + } + + [Theory] + [InlineData("app", null)] + [InlineData("app", "")] + [InlineData("app", " \t")] + [InlineData("group", null)] + [InlineData("group", "")] + [InlineData("group", " \t")] + [InlineData("directory", null)] + [InlineData("directory", "")] + [InlineData("directory", " \t")] + [InlineData("alwaysReady", null)] + [InlineData("environment", null)] + [InlineData("environmentKey", "")] + [InlineData("environmentValue", null)] + public void AssignWorker_InvalidFieldsDoNotClaimAssignment(string field, string? value) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest request = field switch + { + "app" => CreateRequest(functionAppName: value), + "group" => CreateRequest(functionGroupName: value), + "directory" => CreateRequest(functionAppDirectory: value), + "alwaysReady" => CreateRequest(isAlwaysReady: null), + "environment" => new WorkerAssignRequest + { + FunctionAppName = "app", + FunctionGroupName = "group", + FunctionAppDirectory = "app-directory", + IsAlwaysReady = false + }, + "environmentKey" => CreateRequest(environment: new() { [string.Empty] = "private-value" }), + "environmentValue" => CreateRequest(environment: new() { ["SETTING"] = value }), + _ => throw new ArgumentException("Unknown field.", nameof(field)) + }; + + RequestValidationError detail = Assert.Single(AssertValidation(ManagementApiHandlers.AssignWorker(request, manager))); + string expectedTarget = field switch + { + "app" => "functionAppName", + "group" => "functionGroupName", + "directory" => "functionAppDirectory", + "alwaysReady" => "isAlwaysReady", + _ => "environment" + }; + Assert.Equal(expectedTarget, detail.Target); + Assert.Equal(field is "environmentKey" or "environmentValue" ? "InvalidValue" : "Required", detail.Code); + + Assert.Same(before, manager.State); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); + Assert.Equal("other-app", manager.State.FunctionAppName); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AssignWorker_ExplicitBooleanAndEmptyEnvironmentAreAccepted(bool isAlwaysReady) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + + IResult result = ManagementApiHandlers.AssignWorker( + CreateRequest(isAlwaysReady: isAlwaysReady, environment: new()), manager); + + Assert.Equal(200, Assert.IsType(result).StatusCode); + Assert.Equal(before.Revision + 1, manager.State.Revision); + Assert.Equal(WorkerAssignmentState.Ready, manager.State.AssignmentState); + Assert.Equal(WorkerPodStatus.ReadyForRequest, manager.State.PodStatus); + Assert.Equal("app", manager.State.FunctionAppName); + Assert.Equal("group", manager.State.FunctionGroupName); + Assert.Equal(isAlwaysReady, manager.State.IsAlwaysReady); + Assert.Equal(before.SessionId, manager.State.SessionId); + Assert.Equal(before.WorkerId, manager.State.WorkerId); + Assert.Equal(WorkerAssignmentState.Unassigned, before.AssignmentState); + Assert.Null(before.FunctionAppName); + } + + [Fact] + public void AssignWorker_EmptyValuesAndNonemptyWhitespaceKeysAreAccepted() + { + WorkerPodStateManager manager = CreateReadyManager(); + + Assert.IsType(ManagementApiHandlers.AssignWorker( + CreateRequest(environment: new() { ["SETTING"] = string.Empty, [" "] = string.Empty }), manager)); + } + + [Fact] + public void AssignWorker_RecordsEnvironmentWithoutApplyingItToProxyProcess() + { + WorkerPodStateManager manager = CreateReadyManager(); + string setting = $"WORKERPROXY_ASSIGNMENT_TEST_{Guid.NewGuid():N}"; + Assert.Null(Environment.GetEnvironmentVariable(setting)); + + Assert.IsType(ManagementApiHandlers.AssignWorker( + CreateRequest(environment: new() { [setting] = "private-value" }), manager)); + + Assert.Null(Environment.GetEnvironmentVariable(setting)); + Assert.Equal(WorkerAssignmentState.Ready, manager.State.AssignmentState); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AssignWorker_NotReadyDoesNotReserveIdentity(bool attached) + { + WorkerPodStateManager manager = CreateManager(); + if (attached) + { + manager.OnWorkerAttached(1); + } + + WorkerPodState before = manager.State; + AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(), manager), 503, "WorkerNotReady"); + Assert.Same(before, manager.State); + + if (!attached) + { + manager.OnWorkerAttached(1); + } + + manager.OnWorkerStartStream(1, "worker"); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); + Assert.Equal("other-app", manager.State.FunctionAppName); + } + + [Fact] + public void AssignWorker_CopiesEnvironmentAndReplaysEquivalentIdentityRegardlessOfOrder() + { + WorkerPodStateManager manager = CreateReadyManager(); + Dictionary environment = new() { ["SETTING"] = "private-value", ["EMPTY"] = string.Empty }; + WorkerAssignRequest request = CreateRequest(environment: environment); + Assert.IsType(ManagementApiHandlers.AssignWorker(request, manager)); + WorkerPodState assigned = manager.State; + + environment["SETTING"] = "changed-value"; + environment["NEW"] = "new-value"; + WorkerAssignRequest equivalent = CreateRequest(environment: new() + { + ["EMPTY"] = string.Empty, + ["SETTING"] = "private-value" + }); + + Assert.IsType(ManagementApiHandlers.AssignWorker(equivalent, manager)); + AssertError(ManagementApiHandlers.AssignWorker(request, manager), 409, "AssignmentConflict"); + Assert.Same(assigned, manager.State); + } + + [Theory] + [InlineData("app")] + [InlineData("group")] + [InlineData("directory")] + [InlineData("alwaysReady")] + [InlineData("environmentKey")] + [InlineData("environmentValue")] + public void AssignWorker_DifferentIdentityConflictsBeforeAndAfterTermination(string field) + { + WorkerPodStateManager manager = CreateReadyManager(); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + WorkerAssignRequest different = field switch + { + "app" => CreateRequest(functionAppName: "APP"), + "group" => CreateRequest(functionGroupName: "GROUP"), + "directory" => CreateRequest(functionAppDirectory: "APP-DIRECTORY"), + "alwaysReady" => CreateRequest(isAlwaysReady: true), + "environmentKey" => CreateRequest(environment: new() { ["setting"] = "private-value" }), + "environmentValue" => CreateRequest(environment: new() { ["SETTING"] = "PRIVATE-VALUE" }), + _ => throw new ArgumentException("Unknown field.", nameof(field)) + }; + WorkerPodState assigned = manager.State; + + AssertError(ManagementApiHandlers.AssignWorker(different, manager), 409, "AssignmentConflict"); + Assert.Same(assigned, manager.State); + manager.OnSessionTerminated(1); + WorkerPodState terminated = manager.State; + + AssertError(ManagementApiHandlers.AssignWorker(different, manager), 409, "AssignmentConflict"); + AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(), manager), 503, "WorkerTerminated"); + Assert.Same(terminated, manager.State); + } + + [Fact] + public async Task GetInstanceStateAsync_NullRequestReturnsInvalidBodyWithoutWaiting() + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + WorkerPodState initial = manager.State; + + Assert.Equal(new("InvalidBody", "request"), Assert.Single( + AssertValidation(await ManagementApiHandlers.GetInstanceStateAsync(null, manager)))); + + Assert.Same(initial, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetInstanceStateAsync_UnspecifiedRevisionReturnsImmediateImmutableSnapshot(bool assigned) + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + if (assigned) + { + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + } + + WorkerPodState before = manager.State; + Task poll = ManagementApiHandlers.GetInstanceStateAsync(new(), manager); + + Assert.True(poll.IsCompletedSuccessfully); + WorkerInstanceState response = AssertState(await poll, before); + Assert.Same(before, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + + if (assigned) + { + manager.OnSessionTerminated(1); + } + else + { + manager.OnWorkerAttached(1); + } + + Assert.Equal(WorkerInstanceState.FromState(before), response); + Assert.NotEqual(manager.State.Revision, response.RevisionId); + } + + [Fact] + public async Task GetInstanceStateAsync_StaleRevisionReturnsCurrentSnapshotImmediately() + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + manager.OnWorkerAttached(1); + + Task poll = ManagementApiHandlers.GetInstanceStateAsync(new() { LastKnownRevision = 0 }, manager); + + Assert.True(poll.IsCompletedSuccessfully); + AssertState(await poll, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + [Theory] + [InlineData(-1)] + [InlineData(1)] + [InlineData(long.MaxValue)] + public async Task GetInstanceStateAsync_InvalidRevisionReturnsErrorWithoutWaiting(long revision) + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + WorkerPodState initial = manager.State; + + Assert.Equal(new("InvalidRevision", "lastKnownRevision"), Assert.Single( + AssertValidation(await ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = revision }, manager)))); + + Assert.Same(initial, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + [Theory] + [InlineData("attach")] + [InlineData("start")] + [InlineData("assign")] + [InlineData("terminate")] + public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapshot(string transition) + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + if (transition != "attach") + { + manager.OnWorkerAttached(1); + } + + if (transition is "assign" or "terminate") + { + manager.OnWorkerStartStream(1, "worker"); + } + + if (transition == "terminate") + { + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + } + + long revision = manager.State.Revision; + Task poll = ManagementApiHandlers.GetInstanceStateAsync(new() { LastKnownRevision = revision }, manager); + Assert.False(poll.IsCompleted); + Assert.Equal(1, manager.PendingWaiterCount); + + switch (transition) + { + case "attach": + manager.OnWorkerAttached(1); + break; + case "start": + manager.OnWorkerStartStream(1, "worker"); + break; + case "assign": + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + break; + case "terminate": + manager.OnSessionTerminated(1); + break; + default: + throw new ArgumentException("Unknown transition.", nameof(transition)); + } + + WorkerInstanceState response = AssertState(await poll.WaitAsync(TestTimeout), manager.State); + Assert.Equal(revision + 1, response.RevisionId); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task GetInstanceStateAsync_ReplaysAndConflictsDoNotWakePollBeforeSixtySecondTimeout() + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateReadyManager(clock.Provider); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + WorkerPodState assigned = manager.State; + Task poll = ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = assigned.Revision }, manager); + + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other"), manager), + 409, "AssignmentConflict"); + Assert.False(poll.IsCompleted); + Assert.Equal(1, manager.PendingWaiterCount); + PollClock.ScheduledTimer timer = Assert.Single(clock.Timers); + Assert.Equal(TimeSpan.FromSeconds(60), timer.DueTime); + Assert.Equal(Timeout.InfiniteTimeSpan, timer.Period); + + timer.Fire(); + + Assert.Equal(204, Assert.IsType(await poll.WaitAsync(TestTimeout)).StatusCode); + Assert.Same(assigned, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task GetInstanceStateAsync_CancellationPropagatesAndLeavesOtherPollActive() + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + using CancellationTokenSource cancellation = new(); + Task canceledPoll = ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = 0 }, manager, cancellation.Token); + Task activePoll = ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = 0 }, manager); + Assert.Equal(2, manager.PendingWaiterCount); + + cancellation.Cancel(); + OperationCanceledException exception = await Assert.ThrowsAnyAsync( + () => canceledPoll.WaitAsync(TestTimeout)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, manager.PendingWaiterCount); + Assert.False(activePoll.IsCompleted); + manager.OnWorkerAttached(1); + AssertState(await activePoll.WaitAsync(TestTimeout), manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task GetInstanceStateAsync_CancelingOnlyPollRemovesWaiterWithoutChangingState() + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + WorkerPodState initial = manager.State; + using CancellationTokenSource cancellation = new(); + Task poll = ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = 0 }, manager, cancellation.Token); + Assert.Equal(1, manager.PendingWaiterCount); + + cancellation.Cancel(); + OperationCanceledException exception = await Assert.ThrowsAnyAsync( + () => poll.WaitAsync(TestTimeout)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Same(initial, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Theory] + [InlineData(null)] + [InlineData(0L)] + public async Task GetInstanceStateAsync_PreCanceledRequestPropagatesWithoutRegisteringWaiter(long? revision) + { + PollClock clock = new(); + WorkerPodStateManager manager = CreateManager(clock.Provider); + using CancellationTokenSource cancellation = new(); + cancellation.Cancel(); + + OperationCanceledException exception = await Assert.ThrowsAnyAsync( + () => ManagementApiHandlers.GetInstanceStateAsync( + new() { LastKnownRevision = revision }, manager, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + private static WorkerPodStateManager CreateManager(TimeProvider? timeProvider = null) => + new(Options.Create(new WorkerProxyOptions { PodName = "pod" }), timeProvider ?? TimeProvider.System); + + private static WorkerPodStateManager CreateReadyManager(TimeProvider? timeProvider = null) + { + WorkerPodStateManager manager = CreateManager(timeProvider); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + return manager; + } + + private static WorkerAssignRequest CreateRequest( + string? functionAppName = "app", + string? functionGroupName = "group", + bool? isAlwaysReady = false, + string? functionAppDirectory = "app-directory", + Dictionary? environment = null) => + new() + { + FunctionAppName = functionAppName, + FunctionGroupName = functionGroupName, + IsAlwaysReady = isAlwaysReady, + FunctionAppDirectory = functionAppDirectory, + Environment = environment ?? new() { ["SETTING"] = "private-value" } + }; + + private static WorkerApiError AssertError(IResult result, int statusCode, string code) + { + JsonHttpResult json = Assert.IsType>(result); + Assert.Equal(statusCode, json.StatusCode); + WorkerApiErrorResponse response = Assert.IsType(json.Value); + Assert.Equal(code, response.Error.Code); + Assert.False(string.IsNullOrWhiteSpace(response.Error.Detail)); + Assert.DoesNotContain("private-value", response.Error.Detail!); + Assert.DoesNotContain("app-directory", response.Error.Detail!); + return response.Error; + } + + private static IReadOnlyList AssertValidation(IResult result) + { + JsonHttpResult json = Assert.IsType>(result); + Assert.Equal(400, json.StatusCode); + RequestValidationResponse response = Assert.IsType(json.Value); + Assert.NotEmpty(response.Errors); + return response.Errors; + } + + private static WorkerInstanceState AssertState(IResult result, WorkerPodState expected) + { + JsonHttpResult json = Assert.IsType>(result); + Assert.Equal(200, json.StatusCode ?? StatusCodes.Status200OK); + WorkerInstanceState response = Assert.IsType(json.Value); + Assert.Equal(WorkerInstanceState.FromState(expected), response); + return response; + } + + private sealed class PollClock + { + public PollClock() + { + Mock provider = new(); + provider.Setup(clock => clock.CreateTimer( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + { + ScheduledTimer timer = new(callback, state, dueTime, period); + Timers.Add(timer); + return timer.Timer.Object; + }); + Provider = provider.Object; + } + + public TimeProvider Provider { get; } + + public List Timers { get; } = []; + + public void VerifyTimersDisposed() => + Assert.All(Timers, timer => timer.Timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce())); + + public sealed record ScheduledTimer(TimerCallback Callback, object? State, TimeSpan DueTime, TimeSpan Period) + { + public Mock Timer { get; } = new(); + + public void Fire() => Callback(State); + } + } +} diff --git a/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs new file mode 100644 index 0000000000..35e6e25696 --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs @@ -0,0 +1,307 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Azure.Functions.WorkerProxy.Management; +using Azure.Functions.WorkerProxy.State; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.Management; + +public class WorkerProxyJsonContextTests +{ + [Fact] + public void Assignment_RoundTripsWithCamelCaseAndExplicitFalse() + { + WorkerAssignRequest request = new() + { + FunctionAppName = "app", + FunctionGroupName = "group", + FunctionAppDirectory = "private-directory", + IsAlwaysReady = false, + Environment = new Dictionary { ["MixedCase_SETTING"] = string.Empty, ["SECRET"] = "private-value" } + }; + + string json = JsonSerializer.Serialize(request, WorkerProxyJsonContext.Default.WorkerAssignRequest); + using JsonDocument document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + AssertProperties(root, "functionAppName", "functionGroupName", "functionAppDirectory", "isAlwaysReady", "environment"); + Assert.False(root.GetProperty("isAlwaysReady").GetBoolean()); + Assert.Equal(string.Empty, root.GetProperty("environment").GetProperty("MixedCase_SETTING").GetString()); + Assert.Equal("private-value", root.GetProperty("environment").GetProperty("SECRET").GetString()); + + WorkerAssignRequest copy = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + Assert.Equal(request.FunctionAppName, copy.FunctionAppName); + Assert.Equal(request.FunctionGroupName, copy.FunctionGroupName); + Assert.Equal(request.FunctionAppDirectory, copy.FunctionAppDirectory); + Assert.False(copy.IsAlwaysReady); + Assert.Equal(request.Environment.OrderBy(pair => pair.Key), copy.Environment!.OrderBy(pair => pair.Key)); + } + + [Fact] + public void Assignment_PropertyNamesAreCaseInsensitiveButEnvironmentKeysArePreserved() + { + const string json = """ + {"FUNCTIONAPPNAME":"app","FunctionGroupName":"group","FUNCTIONAPPDIRECTORY":"directory", + "ISALWAYSREADY":true,"ENVIRONMENT":{"Key":"one","key":"two"}} + """; + + WorkerAssignRequest request = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + + Assert.Equal("app", request.FunctionAppName); + Assert.Equal("group", request.FunctionGroupName); + Assert.Equal("directory", request.FunctionAppDirectory); + Assert.True(request.IsAlwaysReady); + Assert.Equal(2, request.Environment!.Count); + Assert.Equal("one", request.Environment["Key"]); + Assert.Equal("two", request.Environment["key"]); + } + + [Theory] + [InlineData("{}")] + [InlineData("""{"functionAppName":null,"functionGroupName":null,"functionAppDirectory":null,"isAlwaysReady":null,"environment":null}""")] + public void Assignment_MissingRequiredValuesAreLeftForHandlerValidation(string json) + { + WorkerAssignRequest request = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + + Assert.Null(request.FunctionAppName); + Assert.Null(request.FunctionGroupName); + Assert.Null(request.FunctionAppDirectory); + Assert.Null(request.IsAlwaysReady); + Assert.Null(request.Environment); + Assert.Equal("{}", JsonSerializer.Serialize(request, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + } + + [Fact] + public void Assignment_NullEnvironmentValueIsLeftForHandlerValidation() + { + WorkerAssignRequest request = Assert.IsType(JsonSerializer.Deserialize( + """{"environment":{"SETTING":null}}""", WorkerProxyJsonContext.Default.WorkerAssignRequest)); + + Assert.True(request.Environment!.ContainsKey("SETTING")); + Assert.Null(request.Environment["SETTING"]); + } + + [Theory] + [InlineData("""{"isAlwaysReady":"false"}""")] + [InlineData("""{"isAlwaysReady":0}""")] + [InlineData("""{"functionAppName":123}""")] + [InlineData("""{"functionGroupName":[]}""")] + [InlineData("""{"functionAppDirectory":{}}""")] + [InlineData("""{"environment":[]}""")] + [InlineData("""{"environment":{"SETTING":123}}""")] + [InlineData("[]")] + [InlineData("{")] + public void Assignment_MalformedTypesThrowJsonException(string json) + { + Assert.Throws(() => + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + } + + [Theory] + [InlineData("{}", null)] + [InlineData("""{"lastKnownRevision":null}""", null)] + [InlineData("""{"lastKnownRevision":0}""", 0L)] + [InlineData("""{"LASTKNOWNREVISION":17}""", 17L)] + [InlineData("""{"lastKnownRevision":-1}""", -1L)] + [InlineData("""{"lastKnownRevision":9223372036854775807}""", long.MaxValue)] + public void PollRequest_PreservesNullableAndExactIntegerRevision(string json, long? expected) + { + InstanceStatePollRequest request = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.InstanceStatePollRequest)); + + Assert.Equal(expected, request.LastKnownRevision); + string serialized = JsonSerializer.Serialize(request, WorkerProxyJsonContext.Default.InstanceStatePollRequest); + using JsonDocument document = JsonDocument.Parse(serialized); + if (expected.HasValue) + { + AssertProperties(document.RootElement, "lastKnownRevision"); + Assert.Equal(JsonValueKind.Number, document.RootElement.GetProperty("lastKnownRevision").ValueKind); + Assert.Equal(expected.Value, document.RootElement.GetProperty("lastKnownRevision").GetInt64()); + } + else + { + AssertProperties(document.RootElement); + } + } + + [Theory] + [InlineData("""{"lastKnownRevision":"0"}""")] + [InlineData("""{"lastKnownRevision":true}""")] + [InlineData("""{"lastKnownRevision":1.5}""")] + [InlineData("""{"lastKnownRevision":9223372036854775808}""")] + [InlineData("""{"lastKnownRevision":-9223372036854775809}""")] + [InlineData("""{"lastKnownRevision":{}}""")] + [InlineData("[]")] + [InlineData("{")] + public void PollRequest_MalformedOrOutOfRangeRevisionThrowsJsonException(string json) + { + Assert.Throws(() => + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.InstanceStatePollRequest)); + } + + [Fact] + public void NullRequests_DeserializeAsNullForHandlerValidation() + { + Assert.Null(JsonSerializer.Deserialize("null", WorkerProxyJsonContext.Default.WorkerAssignRequest)); + Assert.Null(JsonSerializer.Deserialize("null", WorkerProxyJsonContext.Default.InstanceStatePollRequest)); + } + + [Fact] + public void InitialState_ContainsOnlyPublicShapeAndOmitsUnknownIdentity() + { + WorkerInstanceState response = WorkerInstanceState.FromState(WorkerPodState.CreateUnassigned("pod")); + + string json = JsonSerializer.Serialize(response, WorkerProxyJsonContext.Default.WorkerInstanceState); + using JsonDocument document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + + AssertProperties(root, "podName", "revisionId", "workerPodState", "functionsContainerType"); + Assert.Equal("pod", root.GetProperty("podName").GetString()); + Assert.Equal(0, root.GetProperty("revisionId").GetInt64()); + Assert.Equal("FunctionsWorkerPod", root.GetProperty("functionsContainerType").GetString()); + AssertProperties(root.GetProperty("workerPodState"), "podStatus"); + Assert.Equal("None", root.GetProperty("workerPodState").GetProperty("podStatus").GetString()); + Assert.Equal(response, JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerInstanceState)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool isAlwaysReady) + { + WorkerPodState state = new( + PodName: "pod", + Revision: long.MaxValue, + SessionId: 123, + IsWorkerAttached: true, + WorkerId: "private-worker", + AssignmentState: WorkerAssignmentState.Ready, + FunctionAppName: "private-app", + FunctionGroupName: "group", + IsAlwaysReady: isAlwaysReady); + WorkerInstanceState response = WorkerInstanceState.FromState(state); + + string json = JsonSerializer.Serialize(response, WorkerProxyJsonContext.Default.WorkerInstanceState); + using JsonDocument document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + JsonElement podState = root.GetProperty("workerPodState"); + + AssertProperties(root, "podName", "revisionId", "workerPodState", "functionsContainerType"); + Assert.Equal("pod", root.GetProperty("podName").GetString()); + Assert.Equal("FunctionsWorkerPod", root.GetProperty("functionsContainerType").GetString()); + Assert.Equal(JsonValueKind.Number, root.GetProperty("revisionId").ValueKind); + Assert.Equal(long.MaxValue, root.GetProperty("revisionId").GetInt64()); + AssertProperties(podState, "podStatus", "functionGroupName", "isAlwaysReady"); + Assert.Equal("ReadyForRequest", podState.GetProperty("podStatus").GetString()); + Assert.Equal("group", podState.GetProperty("functionGroupName").GetString()); + Assert.Equal(isAlwaysReady, podState.GetProperty("isAlwaysReady").GetBoolean()); + Assert.DoesNotContain("private-worker", json); + Assert.DoesNotContain("private-app", json); + Assert.Equal(response, JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerInstanceState)); + } + + [Fact] + public void FromState_RetainedResponseDoesNotChangeWhenSourceStateIsReplaced() + { + WorkerPodState ready = new("pod", 3, 1, true, "worker", WorkerAssignmentState.Ready, "app", "group", false); + WorkerInstanceState retained = WorkerInstanceState.FromState(ready); + WorkerPodState terminated = ready with + { + Revision = 4, + IsWorkerAttached = false, + AssignmentState = WorkerAssignmentState.Failed + }; + + string retainedJson = JsonSerializer.Serialize(retained, WorkerProxyJsonContext.Default.WorkerInstanceState); + string terminatedJson = JsonSerializer.Serialize( + WorkerInstanceState.FromState(terminated), WorkerProxyJsonContext.Default.WorkerInstanceState); + using JsonDocument retainedDocument = JsonDocument.Parse(retainedJson); + using JsonDocument terminatedDocument = JsonDocument.Parse(terminatedJson); + + Assert.Equal(3, retainedDocument.RootElement.GetProperty("revisionId").GetInt64()); + Assert.Equal("ReadyForRequest", retainedDocument.RootElement.GetProperty("workerPodState").GetProperty("podStatus").GetString()); + Assert.Equal(4, terminatedDocument.RootElement.GetProperty("revisionId").GetInt64()); + JsonElement terminatedPodState = terminatedDocument.RootElement.GetProperty("workerPodState"); + Assert.Equal("None", terminatedPodState.GetProperty("podStatus").GetString()); + Assert.Equal("group", terminatedPodState.GetProperty("functionGroupName").GetString()); + Assert.False(terminatedPodState.GetProperty("isAlwaysReady").GetBoolean()); + } + + [Theory] + [InlineData(null)] + [InlineData("The request is invalid.")] + public void ErrorEnvelope_RoundTripsAndOmitsNullDetail(string? detail) + { + WorkerApiErrorResponse response = new(new WorkerApiError("WorkerNotReady", detail)); + + string json = JsonSerializer.Serialize(response, WorkerProxyJsonContext.Default.WorkerApiErrorResponse); + using JsonDocument document = JsonDocument.Parse(json); + AssertProperties(document.RootElement, "error"); + JsonElement error = document.RootElement.GetProperty("error"); + Assert.Equal("WorkerNotReady", error.GetProperty("code").GetString()); + if (detail is null) + { + AssertProperties(error, "code"); + } + else + { + AssertProperties(error, "code", "detail"); + Assert.Equal(detail, error.GetProperty("detail").GetString()); + } + + Assert.Equal(response, JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerApiErrorResponse)); + } + + [Fact] + public void ValidationEnvelope_RoundTripsAllFieldErrorsUsingGeneratedMetadata() + { + RequestValidationError[] details = + [ + new("Required", "functionAppName"), + new("Required", "functionGroupName"), + new("InvalidValue", "environment") + ]; + RequestValidationResponse response = new(details); + + string json = JsonSerializer.Serialize(response, WorkerProxyJsonContext.Default.RequestValidationResponse); + using JsonDocument document = JsonDocument.Parse(json); + AssertProperties(document.RootElement, "errors"); + JsonElement fields = document.RootElement.GetProperty("errors"); + Assert.Equal(details.Length, fields.GetArrayLength()); + for (int index = 0; index < details.Length; index++) + { + AssertProperties(fields[index], "code", "target"); + Assert.Equal(details[index].Code, fields[index].GetProperty("code").GetString()); + Assert.Equal(details[index].Target, fields[index].GetProperty("target").GetString()); + } + + RequestValidationResponse restored = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.RequestValidationResponse)); + Assert.Equal(details, restored.Errors); + } + + [Fact] + public void Context_RecursivelyGeneratesMetadataForNestedResponseTypes() + { + WorkerPodStateResponse state = new(WorkerPodStatus.ReadyForRequest, "group", false); + WorkerApiError error = new("WorkerNotReady"); + + string stateJson = JsonSerializer.Serialize(state, WorkerProxyJsonContext.Default.WorkerPodStateResponse); + string errorJson = JsonSerializer.Serialize(error, WorkerProxyJsonContext.Default.WorkerApiError); + + Assert.Equal(state, JsonSerializer.Deserialize(stateJson, WorkerProxyJsonContext.Default.WorkerPodStateResponse)); + Assert.Equal(error, JsonSerializer.Deserialize(errorJson, WorkerProxyJsonContext.Default.WorkerApiError)); + using JsonDocument stateDocument = JsonDocument.Parse(stateJson); + Assert.Equal("ReadyForRequest", stateDocument.RootElement.GetProperty("podStatus").GetString()); + Assert.False(stateDocument.RootElement.GetProperty("isAlwaysReady").GetBoolean()); + } + + private static void AssertProperties(JsonElement element, params string[] names) => + Assert.Equal(names.OrderBy(name => name), element.EnumerateObject().Select(property => property.Name).OrderBy(name => name)); +} diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs new file mode 100644 index 0000000000..040929a19c --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs @@ -0,0 +1,168 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; +using Grpc.Core; +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests; + +public partial class FunctionRpcRelayTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Relay_UsesInjectedFinalizerOnceIncludingEmptyCapabilities(bool rewrite) + { + Uri? destination = rewrite ? new("http://worker:1234/") : null; + TestCapabilityFinalizer finalizer = new(capabilities => + { + if (rewrite) + { + capabilities["custom-capability"] = "finalized"; + } + + return destination; + }); + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.Replace(ServiceDescriptor.Singleton(finalizer))); + FunctionRpcRelay relay = factory.Services.GetRequiredService(); + using CancellationTokenSource timeout = new(TestTimeout); + await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + await ExchangeAsync(runtime, worker, "attach", timeout.Token); + Assert.Equal(0, finalizer.CallCount); + + StreamingMessage failed = CreateInitResponse("failed-init", null); + failed.WorkerInitResponse.Result.Status = StatusResult.Types.Status.Failure; + await worker.WriteAsync(failed, timeout.Token); + Assert.Equal(failed, await runtime.ReadAsync(timeout.Token)); + Assert.Equal(0, finalizer.CallCount); + + StreamingMessage response = CreateInitResponse("successful-init", null); + response.WorkerInitResponse.Capabilities.Clear(); + StreamingMessage expected = response.Clone(); + if (rewrite) + { + expected.WorkerInitResponse.Capabilities["custom-capability"] = "finalized"; + } + + await worker.WriteAsync(response, timeout.Token); + Assert.Equal(expected, await runtime.ReadAsync(timeout.Token)); + Assert.Equal(1, finalizer.CallCount); + Assert.Equal(destination, relay.WorkerHttpDestination); + + StreamingMessage repeated = CreateInitResponse("repeated-init", "http://ignored:5678/"); + StreamingMessage repeatedExpected = repeated.Clone(); + repeatedExpected.WorkerInitResponse.Capabilities.Clear(); + repeatedExpected.WorkerInitResponse.Capabilities.Add(expected.WorkerInitResponse.Capabilities); + await worker.WriteAsync(repeated, timeout.Token); + Assert.Equal(repeatedExpected, await runtime.ReadAsync(timeout.Token)); + Assert.Equal(destination, relay.WorkerHttpDestination); + Assert.Equal(1, finalizer.CallCount); + } + + [Fact] + public async Task Relay_FinalizerFailureTerminatesAssignedWorkerWithoutForwardingResponse() + { + InvalidOperationException failure = new("Injected finalization failure."); + TestCapabilityFinalizer finalizer = new(capabilities => + { + capabilities["partial"] = "must-not-be-forwarded"; + throw failure; + }); + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.Replace(ServiceDescriptor.Singleton(finalizer))); + FunctionRpcRelay relay = factory.Services.GetRequiredService(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using CancellationTokenSource timeout = new(TestTimeout); + await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + await ExchangeAsync(runtime, worker, "attach", timeout.Token); + WorkerAssignment assignment = CreateWorkerAssignment(); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + Task poll = manager.WaitForChangeAsync(manager.State.Revision, timeout.Token); + + await worker.WriteAsync(CreateInitResponse("init", null), timeout.Token); + + Grpc.Core.RpcException exception = await Assert.ThrowsAsync(() => runtime.ReadAsync(timeout.Token)); + Assert.Equal(StatusCode.Unavailable, exception.StatusCode); + Assert.Equal(StatusCode.Unavailable, await worker.WaitForTerminationAsync(timeout.Token)); + await WaitForReleaseAsync(relay, timeout.Token); + WorkerPodState failed = Assert.IsType((await poll).State); + Assert.False(failed.IsWorkerReady); + Assert.Equal(WorkerAssignmentState.Failed, failed.AssignmentState); + Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(assignment)); + Assert.Null(relay.WorkerHttpDestination); + Assert.Equal(FunctionRpcRelayTerminationReason.Faulted, relay.LastTerminalState?.Reason); + Assert.Same(failure, relay.LastTerminalState?.Exception); + Assert.Equal(1, finalizer.CallCount); + } + + [Fact] + public async Task Relay_ShutdownDuringFinalizationDoesNotRestoreDestinationOrReadiness() + { + using ManualResetEventSlim release = new(initialState: false); + TaskCompletionSource entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource returned = new(TaskCreationOptions.RunContinuationsAsynchronously); + TestCapabilityFinalizer finalizer = new(capabilities => + { + entered.TrySetResult(true); + release.Wait(); + capabilities["late-capability"] = "finalized"; + returned.TrySetResult(true); + return new Uri("http://worker:1234/"); + }); + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.Replace(ServiceDescriptor.Singleton(finalizer))); + FunctionRpcRelay relay = factory.Services.GetRequiredService(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using CancellationTokenSource timeout = new(TestTimeout); + await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + await ExchangeAsync(runtime, worker, "attach", timeout.Token); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + + try + { + await worker.WriteAsync(CreateInitResponse("init", null), timeout.Token); + await entered.Task.WaitAsync(timeout.Token); + await Task.Run(() => relay.StopAsync(timeout.Token), timeout.Token).WaitAsync(timeout.Token); + Assert.Null(relay.WorkerHttpDestination); + Assert.False(manager.State.IsWorkerReady); + Assert.Equal(WorkerAssignmentState.Failed, manager.State.AssignmentState); + } + finally + { + release.Set(); + } + + await returned.Task.WaitAsync(timeout.Token); + await Task.WhenAll(runtime.WaitForTerminationAsync(timeout.Token), worker.WaitForTerminationAsync(timeout.Token)); + Assert.Equal(FunctionRpcRelayTerminationReason.Shutdown, relay.LastTerminalState?.Reason); + Assert.Null(relay.WorkerHttpDestination); + Assert.Equal(WorkerAssignmentState.Failed, manager.State.AssignmentState); + Assert.Equal(4, manager.State.Revision); + } + + private sealed class TestCapabilityFinalizer(Func, Uri?> finalize) : IWorkerCapabilityFinalizer + { + private int _callCount; + + public int CallCount => Volatile.Read(ref _callCount); + + public Uri? FinalizeCapabilities(IDictionary capabilities) + { + Interlocked.Increment(ref _callCount); + return finalize(capabilities); + } + } +} diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs index f18c69b5a4..4aa93f7a4b 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs @@ -81,6 +81,9 @@ public async Task Relay_UnsuccessfulInitialization_DoesNotCaptureOrRewriteCapabi await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); await runtime.WriteAsync(CreateMessage("attach"), timeout.Token); + StreamingMessage start = CreateStartStream(); + await worker.WriteAsync(start, timeout.Token); + Assert.Equal(start, await runtime.ReadAsync(timeout.Token)); StreamingMessage response = CreateInitResponse("failed-init", "http://localhost:1234"); response.WorkerInitResponse.Result = status is { } result ? new() { Status = result } : null; @@ -98,7 +101,7 @@ public async Task Relay_RuntimeMessages_AreNotCapabilityFinalized() using CancellationTokenSource timeout = new(TestTimeout); await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); - await worker.WriteAsync(CreateMessage("attach"), timeout.Token); + await worker.WriteAsync(CreateStartStream(), timeout.Token); StreamingMessage message = CreateInitResponse("runtime-message", "http://localhost:1234"); await runtime.WriteAsync(message, timeout.Token); @@ -113,14 +116,23 @@ public async Task Relay_RuntimeMessages_AreNotCapabilityFinalized() public async Task Relay_FinalizesCapabilitiesOnceOnAnOwnedCopy(string? firstHttpUri) { WorkerProxyOptions options = new() { HttpProxyEndpoint = "https://worker-pod.example:48801/" }; - await using FunctionRpcRelay relay = new(NullLogger.Instance, CreateCapabilityProvider(options)); + await using FunctionRpcRelay relay = new(NullLogger.Instance, CreateCapabilityProvider(options), CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); Channel inbound = Channel.CreateUnbounded(); Channel outbound = Channel.CreateUnbounded(); Task runtimeTask = relay.AttachAsync( - FunctionRpcRelaySide.Runtime, new BlockingStreamReader(), CreateMessageWriter(outbound.Writer), timeout.Token); + FunctionRpcRelaySide.Runtime, new BlockingStreamReader(), CreateMessageWriter(outbound.Writer, message => + { + if (message.WorkerInitResponse is not null) + { + Assert.Equal(firstHttpUri is null ? null : new Uri(firstHttpUri), relay.WorkerHttpDestination); + } + }), timeout.Token); Task workerTask = relay.AttachAsync( FunctionRpcRelaySide.Worker, CreateMessageReader(inbound.Reader), new TestServerStreamWriter(), timeout.Token); + StreamingMessage start = CreateStartStream(); + await inbound.Writer.WriteAsync(start, timeout.Token); + Assert.Equal(start, await outbound.Reader.ReadAsync(timeout.Token)); StreamingMessage first = CreateInitResponse("first-init", firstHttpUri); StreamingMessage expected = first.Clone(); if (firstHttpUri is not null) @@ -206,13 +218,14 @@ public async Task Relay_BlockedCapabilityLogging_DoesNotBlockShutdown() { using BlockingLogger logger = new(); WorkerHttpCapabilityProvider provider = new(Options.Create(new WorkerProxyOptions()), logger); - await using FunctionRpcRelay relay = new(NullLogger.Instance, provider); + await using FunctionRpcRelay relay = new(NullLogger.Instance, provider, CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); Channel inbound = Channel.CreateUnbounded(); Task runtimeTask = relay.AttachAsync( FunctionRpcRelaySide.Runtime, new BlockingStreamReader(), new TestServerStreamWriter(), timeout.Token); Task workerTask = relay.AttachAsync( FunctionRpcRelaySide.Worker, CreateMessageReader(inbound.Reader), new TestServerStreamWriter(), timeout.Token); + await inbound.Writer.WriteAsync(CreateStartStream(), timeout.Token); await inbound.Writer.WriteAsync(CreateInitResponse("init", "invalid"), timeout.Token); try @@ -242,7 +255,9 @@ public async Task Relay_ReplacementSession_DoesNotReusePreviousDestination(strin await using (RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token)) { await ExchangeAsync(runtime, worker, "first", timeout.Token); - await worker.WriteAsync(CreateInitResponse("first-init", "http://localhost:1234"), timeout.Token); + StreamingMessage first = CreateInitResponse("first-init", "http://localhost:1234"); + first.WorkerInitResponse.Capabilities["original-only"] = "true"; + await worker.WriteAsync(first, timeout.Token); await runtime.ReadAsync(timeout.Token); Assert.Equal(new Uri("http://localhost:1234"), relay.WorkerHttpDestination); await runtime.CompleteRequestAsync(timeout.Token); @@ -256,11 +271,15 @@ public async Task Relay_ReplacementSession_DoesNotReusePreviousDestination(strin await ExchangeAsync(replacementRuntime, replacementWorker, "replacement", timeout.Token); Assert.Null(relay.WorkerHttpDestination); - await replacementWorker.WriteAsync(CreateInitResponse("replacement-init", replacementEndpoint), timeout.Token); + StreamingMessage replacement = CreateInitResponse("replacement-init", replacementEndpoint); + replacement.WorkerInitResponse.Capabilities["replacement-only"] = "true"; + await replacementWorker.WriteAsync(replacement, timeout.Token); StreamingMessage response = await replacementRuntime.ReadAsync(timeout.Token); Assert.Equal(replacementEndpoint is null ? null : new Uri(replacementEndpoint), relay.WorkerHttpDestination); Assert.Equal(replacementEndpoint is not null, response.WorkerInitResponse.Capabilities.ContainsKey("HttpUri")); + Assert.False(response.WorkerInitResponse.Capabilities.ContainsKey("original-only")); + Assert.Equal("true", response.WorkerInitResponse.Capabilities["replacement-only"]); } [Fact] @@ -388,17 +407,27 @@ private static IAsyncStreamReader CreateMessageReader(ChannelR reader.SetupGet(value => value.Current).Returns(() => current); reader.Setup(value => value.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellationToken) => { + if (!await messages.WaitToReadAsync(cancellationToken)) + { + return false; + } + current = await messages.ReadAsync(cancellationToken); return true; }); return reader.Object; } - private static IServerStreamWriter CreateMessageWriter(ChannelWriter messages) + private static IServerStreamWriter CreateMessageWriter( + ChannelWriter messages, Action? onWrite = null) { Mock> writer = new(MockBehavior.Strict); writer.Setup(value => value.WriteAsync(It.IsAny(), It.IsAny())) - .Returns((StreamingMessage message, CancellationToken cancellationToken) => messages.WriteAsync(message, cancellationToken).AsTask()); + .Returns((StreamingMessage message, CancellationToken cancellationToken) => + { + onWrite?.Invoke(message); + return messages.WriteAsync(message, cancellationToken).AsTask(); + }); return writer.Object; } } diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs new file mode 100644 index 0000000000..7e80aa6f2d --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs @@ -0,0 +1,229 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; +using Grpc.Core; +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests; + +public partial class FunctionRpcRelayTests +{ + private const string ManagementAssignment = """ + {"functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, + "environment":{"B":"private-value","A":"1"},"functionAppDirectory":"/home/site/wwwroot"} + """; + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ManagementApis_ObserveNormalStartupAssignmentAndTerminalFailure(bool runtimeFirst) + { + const string proxyEndpoint = "http://worker-pod:28080/"; + await using WorkerProxyWebApplicationFactory factory = CreateHttpCapabilityFactory(proxyEndpoint); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using HttpClient management = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + await AssertReadinessAsync(management, "/admin/instance/ready", HttpStatusCode.OK, timeout.Token); + await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); + using (HttpResponseMessage notReady = await PostManagementJsonAsync( + management, "/admin/worker/assign", ManagementAssignment, timeout.Token)) + { + await AssertManagementErrorAsync(notReady, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); + } + + Assert.Equal(0, manager.State.Revision); + StreamingMessage init = new() + { + RequestId = "runtime-init", + WorkerInitRequest = new() { HostVersion = "test-host", FunctionAppDirectory = "runtime-authoritative-directory" } + }; + RelayClient? runtime = null; + try + { + if (runtimeFirst) + { + runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await runtime.WriteAsync(init, timeout.Token); + await WaitForAttachmentAsync(factory.Services.GetRequiredService(), + FunctionRpcRelaySide.Runtime, timeout.Token); + await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); + } + + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + StreamingMessage start = CreateStartStream(); + await worker.WriteAsync(start, timeout.Token); + while (!manager.State.IsWorkerReady) + { + await Task.Delay(TimeSpan.FromMilliseconds(10), timeout.Token); + } + + await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.OK, timeout.Token); + using (JsonDocument unassigned = await ReadManagementStateAsync(management, "{}", timeout.Token)) + { + Assert.Equal(2, unassigned.RootElement.GetProperty("revisionId").GetInt64()); + Assert.Equal("None", unassigned.RootElement.GetProperty("workerPodState").GetProperty("podStatus").GetString()); + } + + Task assignmentPoll = PostManagementJsonAsync( + management, "/admin/infra/instanceState", "{\"lastKnownRevision\":2}", timeout.Token); + await WaitForManagementPollAsync(manager, timeout.Token); + using (HttpResponseMessage assignment = await PostManagementJsonAsync( + management, "/admin/worker/assign", ManagementAssignment, timeout.Token)) + { + Assert.Equal(HttpStatusCode.OK, assignment.StatusCode); + Assert.Empty(await assignment.Content.ReadAsByteArrayAsync(timeout.Token)); + } + + using (HttpResponseMessage changed = await assignmentPoll) + { + Assert.Equal(HttpStatusCode.OK, changed.StatusCode); + string body = await changed.Content.ReadAsStringAsync(timeout.Token); + using JsonDocument state = JsonDocument.Parse(body); + Assert.Equal("FunctionsWorkerPod", state.RootElement.GetProperty("functionsContainerType").GetString()); + Assert.Equal("test-worker-pod", state.RootElement.GetProperty("podName").GetString()); + Assert.Equal(3, state.RootElement.GetProperty("revisionId").GetInt64()); + JsonElement pod = state.RootElement.GetProperty("workerPodState"); + Assert.Equal("ReadyForRequest", pod.GetProperty("podStatus").GetString()); + Assert.Equal("test-group", pod.GetProperty("functionGroupName").GetString()); + Assert.False(pod.GetProperty("isAlwaysReady").GetBoolean()); + Assert.DoesNotContain("private-value", body); + Assert.DoesNotContain("environment", body); + Assert.DoesNotContain("runtimePodName", body); + } + + string replayBody = ManagementAssignment.Replace( + "\"B\":\"private-value\",\"A\":\"1\"", "\"A\":\"1\",\"B\":\"private-value\"", StringComparison.Ordinal); + using (HttpResponseMessage replay = await PostManagementJsonAsync( + management, "/admin/worker/assign", replayBody, timeout.Token)) + { + Assert.Equal(HttpStatusCode.OK, replay.StatusCode); + Assert.Empty(await replay.Content.ReadAsByteArrayAsync(timeout.Token)); + } + + string conflictingBody = ManagementAssignment.Replace("test-group", "other-group", StringComparison.Ordinal); + using (HttpResponseMessage conflict = await PostManagementJsonAsync( + management, "/admin/worker/assign", conflictingBody, timeout.Token)) + { + await AssertManagementErrorAsync(conflict, HttpStatusCode.Conflict, "AssignmentConflict", timeout.Token); + } + + Assert.Equal(3, manager.State.Revision); + if (runtime is null) + { + runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await runtime.WriteAsync(init, timeout.Token); + } + + Assert.Equal(start, await runtime.ReadAsync(timeout.Token)); + Assert.Equal(init, await worker.ReadAsync(timeout.Token)); + StreamingMessage initialized = CreateInitResponse(init.RequestId, "http://localhost:1234/"); + StreamingMessage expected = initialized.Clone(); + expected.WorkerInitResponse.Capabilities["HttpUri"] = proxyEndpoint; + await worker.WriteAsync(initialized, timeout.Token); + Assert.Equal(expected, await runtime.ReadAsync(timeout.Token)); + Assert.Equal(3, manager.State.Revision); + + Task terminationPoll = PostManagementJsonAsync( + management, "/admin/infra/instanceState", "{\"lastKnownRevision\":3}", timeout.Token); + await WaitForManagementPollAsync(manager, timeout.Token); + await worker.CompleteRequestAsync(timeout.Token); + Assert.Equal(StatusCode.Unavailable, await runtime.WaitForTerminationAsync(timeout.Token)); + Assert.Equal(StatusCode.Unavailable, await worker.WaitForTerminationAsync(timeout.Token)); + using (HttpResponseMessage changed = await terminationPoll) + { + Assert.Equal(HttpStatusCode.OK, changed.StatusCode); + using JsonDocument state = JsonDocument.Parse(await changed.Content.ReadAsStringAsync(timeout.Token)); + Assert.Equal(4, state.RootElement.GetProperty("revisionId").GetInt64()); + JsonElement pod = state.RootElement.GetProperty("workerPodState"); + Assert.Equal("None", pod.GetProperty("podStatus").GetString()); + Assert.Equal("test-group", pod.GetProperty("functionGroupName").GetString()); + Assert.False(pod.GetProperty("isAlwaysReady").GetBoolean()); + } + + await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); + await AssertReadinessAsync(management, "/admin/instance/ready", HttpStatusCode.OK, timeout.Token); + using HttpResponseMessage terminalReplay = await PostManagementJsonAsync( + management, "/admin/worker/assign", ManagementAssignment, timeout.Token); + await AssertManagementErrorAsync(terminalReplay, HttpStatusCode.ServiceUnavailable, "WorkerTerminated", timeout.Token); + using JsonDocument current = await ReadManagementStateAsync(management, "{\"lastKnownRevision\":2}", timeout.Token); + Assert.Equal(4, current.RootElement.GetProperty("revisionId").GetInt64()); + } + finally + { + if (runtime is not null) + { + await runtime.DisposeAsync(); + } + } + } + + [Fact] + public async Task ManagementApis_InvalidWorkerStartupNeverBecomesReady() + { + await using WorkerProxyWebApplicationFactory factory = CreateFactory(); + using HttpClient management = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + + Grpc.Core.RpcException exception = await worker.WriteAndReadRejectionAsync(CreateMessage("not-start-stream"), timeout.Token); + + Assert.Equal(StatusCode.Unavailable, exception.StatusCode); + await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); + using HttpResponseMessage assignment = await PostManagementJsonAsync( + management, "/admin/worker/assign", ManagementAssignment, timeout.Token); + await AssertManagementErrorAsync(assignment, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); + } + + private static async Task PostManagementJsonAsync( + HttpClient client, string path, string body, CancellationToken cancellationToken) + { + using StringContent content = new(body, Encoding.UTF8, "application/json"); + return await client.PostAsync(path, content, cancellationToken); + } + + private static async Task ReadManagementStateAsync( + HttpClient client, string body, CancellationToken cancellationToken) + { + using HttpResponseMessage response = await PostManagementJsonAsync( + client, "/admin/infra/instanceState", body, cancellationToken); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + return JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + } + + private static async Task AssertReadinessAsync( + HttpClient client, string path, HttpStatusCode statusCode, CancellationToken cancellationToken) + { + using HttpResponseMessage response = await client.GetAsync(path, cancellationToken); + Assert.Equal(statusCode, response.StatusCode); + Assert.Empty(await response.Content.ReadAsByteArrayAsync(cancellationToken)); + } + + private static async Task AssertManagementErrorAsync( + HttpResponseMessage response, HttpStatusCode statusCode, string code, CancellationToken cancellationToken) + { + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); + using JsonDocument json = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + Assert.Equal("error", Assert.Single(json.RootElement.EnumerateObject()).Name); + Assert.Equal(code, json.RootElement.GetProperty("error").GetProperty("code").GetString()); + } + + private static async Task WaitForManagementPollAsync(WorkerPodStateManager manager, CancellationToken cancellationToken) + { + while (manager.PendingWaiterCount == 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken); + } + } +} diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs new file mode 100644 index 0000000000..ae46146dd5 --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs @@ -0,0 +1,338 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; +using Grpc.Core; +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests; + +public partial class FunctionRpcRelayTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Relay_ReadinessRequiresStartStreamButNotRuntimeInitialization(bool runtimeFirst) + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + Channel runtimeInbound = Channel.CreateUnbounded(); + Channel workerInbound = Channel.CreateUnbounded(); + Channel runtimeOutbound = Channel.CreateUnbounded(); + Channel workerOutbound = Channel.CreateUnbounded(); + Task? runtimeTask = null; + + if (runtimeFirst) + { + runtimeTask = relay.AttachAsync(FunctionRpcRelaySide.Runtime, + CreateMessageReader(runtimeInbound.Reader), CreateMessageWriter(runtimeOutbound.Writer), timeout.Token); + } + + Assert.Equal(0, manager.State.Revision); + Assert.False(manager.State.IsWorkerReady); + Assert.Equal(WorkerAssignmentResult.WorkerNotReady, manager.Assign(CreateWorkerAssignment())); + + Task workerTask = relay.AttachAsync(FunctionRpcRelaySide.Worker, + CreateMessageReader(workerInbound.Reader), CreateMessageWriter(workerOutbound.Writer), timeout.Token); + WorkerPodState attached = manager.State; + Assert.Equal(1, attached.Revision); + Assert.True(attached.IsWorkerAttached); + Assert.False(attached.IsWorkerReady); + Assert.Null(attached.WorkerId); + + Task poll = manager.WaitForChangeAsync(attached.Revision, timeout.Token); + StreamingMessage start = CreateStartStream(); + await workerInbound.Writer.WriteAsync(start, timeout.Token); + WorkerPodState ready = Assert.IsType((await poll).State); + Assert.Equal(2, ready.Revision); + Assert.True(ready.IsWorkerReady); + Assert.Equal(start.StartStream.WorkerId, ready.WorkerId); + Assert.Equal(WorkerPodStatus.None, ready.PodStatus); + + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + Assert.Equal(WorkerPodStatus.ReadyForRequest, manager.State.PodStatus); + Assert.Equal(3, manager.State.Revision); + Assert.False(workerOutbound.Reader.TryRead(out _)); + + runtimeTask ??= relay.AttachAsync(FunctionRpcRelaySide.Runtime, + CreateMessageReader(runtimeInbound.Reader), CreateMessageWriter(runtimeOutbound.Writer), timeout.Token); + Assert.Same(start, await runtimeOutbound.Reader.ReadAsync(timeout.Token)); + + StreamingMessage init = new() + { + RequestId = "runtime-init", + WorkerInitRequest = new() { HostVersion = "test-host" } + }; + await runtimeInbound.Writer.WriteAsync(init, timeout.Token); + Assert.Same(init, await workerOutbound.Reader.ReadAsync(timeout.Token)); + Assert.False(workerOutbound.Reader.TryRead(out _)); + Assert.Equal(3, manager.State.Revision); + + await relay.StopAsync(timeout.Token); + await Task.WhenAll(runtimeTask, workerTask).WaitAsync(timeout.Token); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" \t")] + public async Task Relay_InvalidFirstWorkerMessageFaultsWithoutReadinessOrForwarding(string? workerId) + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + Channel outbound = Channel.CreateUnbounded(); + StreamingMessage first = workerId is null + ? CreateMessage("not-start-stream") + : new() { StartStream = new() { WorkerId = workerId } }; + + Task runtimeTask = relay.AttachAsync(FunctionRpcRelaySide.Runtime, + new BlockingStreamReader(), CreateMessageWriter(outbound.Writer), timeout.Token); + Task workerTask = relay.AttachAsync(FunctionRpcRelaySide.Worker, + new SingleMessageThenBlockStreamReader(first), new TestServerStreamWriter(), timeout.Token); + FunctionRpcRelayTerminalState[] terminalStates = await Task.WhenAll(runtimeTask, workerTask).WaitAsync(timeout.Token); + + Assert.All(terminalStates, state => + { + Assert.Equal(FunctionRpcRelayTerminationReason.Faulted, state.Reason); + Assert.Equal(FunctionRpcRelaySide.Worker, state.Side); + Assert.IsType(state.Exception); + }); + Assert.False(outbound.Reader.TryRead(out _)); + Assert.False(manager.State.IsWorkerAttached); + Assert.False(manager.State.IsWorkerReady); + Assert.Null(manager.State.WorkerId); + Assert.Equal(WorkerAssignmentState.Unassigned, manager.State.AssignmentState); + Assert.Equal(2, manager.State.Revision); + } + + [Fact] + public async Task Relay_DuplicateWorkerDoesNotChangeReadiness() + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + Task workerTask = relay.AttachAsync(FunctionRpcRelaySide.Worker, + new SingleMessageThenBlockStreamReader(CreateStartStream()), new TestServerStreamWriter(), timeout.Token); + WorkerPodState ready = manager.State; + Assert.True(ready.IsWorkerReady); + + FunctionRpcRelayAttachmentException exception = await Assert.ThrowsAsync(() => + relay.AttachAsync(FunctionRpcRelaySide.Worker, + new SingleMessageThenBlockStreamReader(CreateMessage("invalid-duplicate")), new TestServerStreamWriter(), timeout.Token)); + + Assert.Equal(FunctionRpcRelayAttachmentFailure.Duplicate, exception.Failure); + Assert.Same(ready, manager.State); + await relay.StopAsync(timeout.Token); + await workerTask.WaitAsync(timeout.Token); + } + + [Theory] + [InlineData("worker-close", nameof(FunctionRpcRelayTerminationReason.PeerClosed))] + [InlineData("runtime-close", nameof(FunctionRpcRelayTerminationReason.PeerClosed))] + [InlineData("worker-fault", nameof(FunctionRpcRelayTerminationReason.Faulted))] + [InlineData("worker-cancel", nameof(FunctionRpcRelayTerminationReason.Canceled))] + [InlineData("runtime-cancel", nameof(FunctionRpcRelayTerminationReason.Canceled))] + [InlineData("shutdown", nameof(FunctionRpcRelayTerminationReason.Shutdown))] + public async Task Relay_TerminationWithdrawsReadinessBeforeBlockedWriterReleases( + string trigger, string expectedReason) + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + using CancellationTokenSource runtimeCancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); + using CancellationTokenSource workerCancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); + Channel runtimeInbound = Channel.CreateUnbounded(); + Channel workerInbound = Channel.CreateUnbounded(); + BlockingServerStreamWriter blockingWriter = new(); + Task runtimeTask = relay.AttachAsync(FunctionRpcRelaySide.Runtime, + CreateMessageReader(runtimeInbound.Reader), blockingWriter, runtimeCancellation.Token); + Task workerTask = relay.AttachAsync(FunctionRpcRelaySide.Worker, + CreateMessageReader(workerInbound.Reader), new TestServerStreamWriter(), workerCancellation.Token); + Task stopTask = Task.CompletedTask; + + try + { + await workerInbound.Writer.WriteAsync(CreateStartStream(), timeout.Token); + await blockingWriter.WriteEntered.WaitAsync(timeout.Token); + WorkerAssignment assignment = CreateWorkerAssignment(); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + WorkerPodState assigned = manager.State; + Task poll = manager.WaitForChangeAsync(assigned.Revision, timeout.Token); + + switch (trigger) + { + case "worker-close": + workerInbound.Writer.Complete(); + break; + case "runtime-close": + runtimeInbound.Writer.Complete(); + break; + case "worker-fault": + workerInbound.Writer.Complete(new IOException("Injected worker failure.")); + break; + case "worker-cancel": + workerCancellation.Cancel(); + break; + case "runtime-cancel": + runtimeCancellation.Cancel(); + break; + case "shutdown": + stopTask = relay.StopAsync(timeout.Token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(trigger)); + } + + WorkerPodState failed = Assert.IsType((await poll).State); + Assert.False(failed.IsWorkerReady); + Assert.False(failed.IsWorkerAttached); + Assert.Equal(WorkerPodStatus.None, failed.PodStatus); + Assert.Equal(WorkerAssignmentState.Failed, failed.AssignmentState); + Assert.Equal(assigned.Revision + 1, failed.Revision); + Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(assignment)); + Assert.False(runtimeTask.IsCompleted); + Assert.True(relay.IsAttached(FunctionRpcRelaySide.Runtime)); + } + finally + { + blockingWriter.Release(); + } + + FunctionRpcRelayTerminalState[] terminalStates = await Task.WhenAll(runtimeTask, workerTask).WaitAsync(timeout.Token); + Assert.All(terminalStates, state => Assert.Equal(expectedReason, state.Reason.ToString())); + await stopTask.WaitAsync(timeout.Token); + Assert.Equal(4, manager.State.Revision); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Relay_ReplacementReadinessRespectsTerminalAssignment(bool assigned) + { + await using WorkerProxyWebApplicationFactory factory = CreateFactory(); + FunctionRpcRelay relay = factory.Services.GetRequiredService(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using CancellationTokenSource timeout = new(TestTimeout); + WorkerAssignment assignment = CreateWorkerAssignment(); + await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + await ExchangeAsync(runtime, worker, "original", timeout.Token); + Assert.True(manager.State.IsWorkerReady); + + if (assigned) + { + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + } + + await worker.CompleteRequestAsync(timeout.Token); + await Task.WhenAll(runtime.WaitForTerminationAsync(timeout.Token), worker.WaitForTerminationAsync(timeout.Token)); + await WaitForReleaseAsync(relay, timeout.Token); + WorkerPodState terminated = manager.State; + Assert.False(terminated.IsWorkerReady); + + await using RelayClient replacementRuntime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); + await using RelayClient replacementWorker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); + await ExchangeAsync(replacementRuntime, replacementWorker, "replacement", timeout.Token); + + if (assigned) + { + Assert.Same(terminated, manager.State); + Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(assignment)); + } + else + { + Assert.True(manager.State.IsWorkerReady); + Assert.True(manager.State.SessionId > terminated.SessionId); + Assert.Equal(terminated.Revision + 2, manager.State.Revision); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + } + } + + [Fact] + public async Task Relay_RuntimeOnlyTerminationDoesNotPreventNextWorkerSession() + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + using CancellationTokenSource runtimeCancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); + WorkerPodState initial = manager.State; + Task runtimeTask = relay.AttachAsync(FunctionRpcRelaySide.Runtime, + new BlockingStreamReader(), new TestServerStreamWriter(), runtimeCancellation.Token); + Assert.Same(initial, manager.State); + + runtimeCancellation.Cancel(); + FunctionRpcRelayTerminalState terminated = await runtimeTask.WaitAsync(timeout.Token); + Assert.Same(initial, manager.State); + + Task workerTask = relay.AttachAsync(FunctionRpcRelaySide.Worker, + new SingleMessageThenBlockStreamReader(CreateStartStream()), new TestServerStreamWriter(), timeout.Token); + Assert.True(manager.State.IsWorkerReady); + Assert.True(manager.State.SessionId > terminated.SessionId); + Assert.Equal(2, manager.State.Revision); + await relay.StopAsync(timeout.Token); + await workerTask.WaitAsync(timeout.Token); + } + + [Fact] + public async Task Relay_DelayedStartStreamCannotChangeReplacementReadiness() + { + WorkerPodStateManager manager = CreatePodStateManager(); + await using FunctionRpcRelay relay = CreateInProcessRelay(manager); + using CancellationTokenSource timeout = new(TestTimeout); + using CancellationTokenSource runtimeCancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); + TaskCompletionSource delayedRead = new(); + Mock> oldReader = new(MockBehavior.Strict); + oldReader.Setup(reader => reader.MoveNext(It.IsAny())).Returns(delayedRead.Task); + oldReader.SetupGet(reader => reader.Current).Returns(new StreamingMessage + { + StartStream = new() { WorkerId = "old-worker" } + }); + + // Register the read without a test synchronization context. Completing delayedRead then processes it inline. + Task oldWorker = await Task.Factory.StartNew( + () => relay.AttachAsync(FunctionRpcRelaySide.Worker, oldReader.Object, new TestServerStreamWriter(), timeout.Token), + timeout.Token, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); + Task oldRuntime = relay.AttachAsync(FunctionRpcRelaySide.Runtime, + new BlockingStreamReader(), new TestServerStreamWriter(), runtimeCancellation.Token); + + try + { + runtimeCancellation.Cancel(); + await Task.WhenAll(oldRuntime, oldWorker).WaitAsync(timeout.Token); + Assert.False(manager.State.IsWorkerReady); + + Task replacement = relay.AttachAsync(FunctionRpcRelaySide.Worker, + new SingleMessageThenBlockStreamReader(CreateStartStream()), new TestServerStreamWriter(), timeout.Token); + WorkerPodState ready = manager.State; + Assert.True(ready.IsWorkerReady); + + delayedRead.SetResult(true); + oldReader.VerifyGet(reader => reader.Current, Times.Once); + Assert.Same(ready, manager.State); + Assert.Equal("test-worker", manager.State.WorkerId); + + await relay.StopAsync(timeout.Token); + await replacement.WaitAsync(timeout.Token); + } + finally + { + delayedRead.TrySetResult(true); + } + } + + private static WorkerAssignment CreateWorkerAssignment() + => new("test-app", "test-group", isAlwaysReady: false, + environment: new Dictionary(), functionAppDirectory: "/home/site/wwwroot"); +} diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs index a27c145cff..62b6f92be6 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Azure.Functions.WorkerProxy.Http; using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; using Google.Protobuf; using Grpc.Core; using Grpc.Net.Client; @@ -46,7 +47,7 @@ public async Task FunctionRpc_IsAvailableOnBothRpcListeners(bool runtimeListener using CancellationTokenSource timeout = new(TestTimeout); await using RelayClient client = CreateClient(factory, side, timeout.Token); - await client.WriteAsync(CreateMessage("attach"), timeout.Token); + await client.WriteAsync(side == FunctionRpcRelaySide.Worker ? CreateStartStream("attach") : CreateMessage("attach"), timeout.Token); await WaitForAttachmentAsync(relay, side, timeout.Token); Assert.True(relay.IsAttached(side)); @@ -106,7 +107,7 @@ public async Task Relay_ConcurrentBidirectionalProducersPreservePerSideOrdering( await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); IReadOnlyList runtimeMessages = CreateMessages("runtime", count: 64); - IReadOnlyList workerMessages = CreateMessages("worker", count: 64); + IReadOnlyList workerMessages = [CreateStartStream("worker-start"), .. CreateMessages("worker", count: 64)]; Task runtimeWrites = runtime.WriteAllAsync(runtimeMessages, timeout.Token); Task workerWrites = worker.WriteAllAsync(workerMessages, timeout.Token); @@ -134,7 +135,7 @@ public async Task Relay_MessagesQueuedBeforePeerConnectsAreDeliveredInOrder() Assert.False(relay.IsAttached(FunctionRpcRelaySide.Worker)); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); - await worker.WriteAsync(CreateMessage("worker-attach"), timeout.Token); + await worker.WriteAsync(CreateStartStream("worker-attach"), timeout.Token); await WaitForAttachmentAsync(relay, FunctionRpcRelaySide.Worker, timeout.Token); IReadOnlyList receivedMessages = await worker.ReadAsync(sentMessages.Count, timeout.Token); @@ -151,7 +152,7 @@ public async Task Relay_RejectsDuplicateSideAttachment(bool duplicateRuntime) FunctionRpcRelay relay = factory.Services.GetRequiredService(); using CancellationTokenSource timeout = new(TestTimeout); await using RelayClient first = CreateClient(factory, side, timeout.Token); - await first.WriteAsync(CreateMessage("first"), timeout.Token); + await first.WriteAsync(side == FunctionRpcRelaySide.Worker ? CreateStartStream("first") : CreateMessage("first"), timeout.Token); await WaitForAttachmentAsync(relay, side, timeout.Token); await using RelayClient duplicate = CreateClient(factory, side, timeout.Token); @@ -200,11 +201,17 @@ public async Task Relay_ReconnectDuringSessionTeardownReturnsUnavailable() await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); await ExchangeAsync(runtime, worker, "teardown", timeout.Token); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + Task poll = manager.WaitForChangeAsync(manager.State.Revision, timeout.Token); await runtime.CompleteRequestAsync(timeout.Token); try { await logger.LogEntered.WaitAsync(timeout.Token); + WorkerPodState failed = Assert.IsType((await poll).State); + Assert.False(failed.IsWorkerReady); + Assert.Equal(WorkerAssignmentState.Failed, failed.AssignmentState); await using RelayClient reconnect = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); GrpcRpcException exception = await reconnect.WriteAndReadRejectionAsync(CreateMessage("reconnect"), timeout.Token); @@ -275,7 +282,7 @@ public async Task Relay_FirstReadFaultStopsBothStreamOperationsAndPreservesFault public async Task Relay_CanceledStopWaitDoesNotCancelSharedStop() { using BlockingLogger logger = new(); - FunctionRpcRelay relay = new(logger, CreateCapabilityProvider()); + FunctionRpcRelay relay = new(logger, CreateCapabilityProvider(), CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); using CancellationTokenSource stopCancellation = new(); BlockingServerStreamWriter blockingWriter = new(); @@ -385,7 +392,7 @@ public async Task Relay_ConcurrentStopsJoinSharedCompletion() public async Task Relay_ShutdownAllowsSessionClearBeforeCancellation() { using BlockingLogger logger = new(); - FunctionRpcRelay relay = new(logger, CreateCapabilityProvider()); + FunctionRpcRelay relay = new(logger, CreateCapabilityProvider(), CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); Task runtimeTask = relay.AttachAsync(FunctionRpcRelaySide.Runtime, new BlockingStreamReader(), new TestServerStreamWriter(), timeout.Token); @@ -417,11 +424,17 @@ private static WorkerProxyWebApplicationFactory CreateFactory() return new WorkerProxyWebApplicationFactory(); } - private static FunctionRpcRelay CreateInProcessRelay() + private static FunctionRpcRelay CreateInProcessRelay(WorkerPodStateManager? stateManager = null) { - return new FunctionRpcRelay(NullLogger.Instance, CreateCapabilityProvider()); + return new FunctionRpcRelay(NullLogger.Instance, CreateCapabilityProvider(), stateManager ?? CreatePodStateManager()); } + private static WorkerPodStateManager CreatePodStateManager() + => new(Options.Create(new WorkerProxyOptions { PodName = "test-worker-pod" }), TimeProvider.System); + + private static StreamingMessage CreateStartStream(string requestId = "worker-start") + => new() { RequestId = requestId, StartStream = new() { WorkerId = "test-worker" } }; + private static WorkerHttpCapabilityProvider CreateCapabilityProvider(WorkerProxyOptions? options = null) { return new(Options.Create(options ?? new()), NullLogger.Instance); @@ -473,9 +486,12 @@ private static async Task ExchangeAsync(RelayClient runtime, RelayClient worker, { StreamingMessage runtimeMessage = CreateMessage($"{requestIdPrefix}-runtime"); StreamingMessage workerMessage = CreateMessage($"{requestIdPrefix}-worker"); - await Task.WhenAll(runtime.WriteAsync(runtimeMessage, cancellationToken), worker.WriteAsync(workerMessage, cancellationToken)); + StreamingMessage start = CreateStartStream($"{requestIdPrefix}-start"); + await Task.WhenAll(runtime.WriteAsync(runtimeMessage, cancellationToken), worker.WriteAsync(start, cancellationToken)); Assert.Equal(runtimeMessage, await worker.ReadAsync(cancellationToken)); + Assert.Equal(start, await runtime.ReadAsync(cancellationToken)); + await worker.WriteAsync(workerMessage, cancellationToken); Assert.Equal(workerMessage, await runtime.ReadAsync(cancellationToken)); } diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs new file mode 100644 index 0000000000..0823821d9b --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs @@ -0,0 +1,98 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Azure.Functions.WorkerProxy.State; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.State; + +public class WorkerAssignmentTests +{ + [Fact] + public void Equality_IgnoresEnvironmentOrderAndCopiesInput() + { + Dictionary environment = new() { ["A"] = "one", ["B"] = "two" }; + WorkerAssignment assignment = Create(environment); + WorkerAssignment reordered = Create(new Dictionary { ["B"] = "two", ["A"] = "one" }); + + environment["A"] = "changed"; + environment.Add("C", "three"); + + Assert.True(assignment.IsEquivalentTo(reordered)); + Assert.True(reordered.IsEquivalentTo(assignment)); + Assert.Equal("one", assignment.Environment["A"]); + Assert.Equal(2, assignment.Environment.Count); + } + + [Theory] + [InlineData("app")] + [InlineData("group")] + [InlineData("alwaysReady")] + [InlineData("directory")] + [InlineData("key")] + [InlineData("value")] + [InlineData("count")] + public void Equality_UsesEveryFieldAndOrdinalStrings(string changedField) + { + WorkerAssignment original = Create(new Dictionary { ["KEY"] = "Value" }); + Dictionary environment = new(StringComparer.OrdinalIgnoreCase) + { + [changedField == "key" ? "key" : "KEY"] = changedField == "value" ? "value" : "Value" + }; + if (changedField == "count") + { + environment.Add("EXTRA", "Value"); + } + + WorkerAssignment changed = new( + changedField == "app" ? "APP" : "app", + changedField == "group" ? "HTTP" : "http", + changedField == "alwaysReady", + environment, + changedField == "directory" ? "/HOME/site/wwwroot" : "/home/site/wwwroot"); + + Assert.False(original.IsEquivalentTo(changed)); + Assert.False(changed.IsEquivalentTo(original)); + } + + [Fact] + public void Equality_PreservesDistinctEnvironmentKeyCasing() + { + WorkerAssignment assignment = Create(new Dictionary { ["KEY"] = "one", ["key"] = "two" }); + + Assert.Equal(2, assignment.Environment.Count); + Assert.Equal("one", assignment.Environment["KEY"]); + Assert.Equal("two", assignment.Environment["key"]); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void Construction_RejectsMissingIdentity(string? value) + { + Dictionary environment = []; + Assert.ThrowsAny(() => new WorkerAssignment(value!, "http", false, environment, "/app")); + Assert.ThrowsAny(() => new WorkerAssignment("app", value!, false, environment, "/app")); + Assert.ThrowsAny(() => new WorkerAssignment("app", "http", false, environment, value!)); + } + + [Fact] + public void Construction_RejectsInvalidEnvironment() + { + Assert.Throws(() => Create(null!)); + Assert.Throws(() => Create(new Dictionary { [string.Empty] = "value" })); + Assert.Throws(() => Create(new Dictionary { ["KEY"] = null! })); + } + + [Fact] + public void Equality_RejectsNull() + { + Assert.Throws(() => Create(new Dictionary()).IsEquivalentTo(null!)); + } + + private static WorkerAssignment Create(IReadOnlyDictionary environment) + => new("app", "http", false, environment, "/home/site/wwwroot"); +} diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs new file mode 100644 index 0000000000..71c9a2a89d --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs @@ -0,0 +1,375 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.State; +using Moq; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.State; + +public partial class WorkerPodStateManagerTests +{ + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10); + + [Fact] + public async Task Poll_StaleRevisionReturnsCurrentSnapshotWithoutWaiting() + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + WorkerPodState state = manager.State; + + Task poll = manager.WaitForChangeAsync(lastKnownRevision: 0); + + Assert.True(poll.IsCompletedSuccessfully); + WorkerStatePollResult result = await poll; + Assert.True(result.HasChanged); + Assert.Same(state, result.State); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + [Theory] + [InlineData(-1)] + [InlineData(1)] + [InlineData(long.MaxValue)] + public async Task Poll_InvalidRevisionDoesNotRegisterWaiter(long lastKnownRevision) + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + WorkerPodState initial = manager.State; + + ArgumentOutOfRangeException exception = await Assert.ThrowsAsync( + () => manager.WaitForChangeAsync(lastKnownRevision)); + + Assert.Equal("lastKnownRevision", exception.ParamName); + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + Assert.Same(initial, manager.State); + } + + [Theory] + [InlineData("attach")] + [InlineData("start")] + [InlineData("assign")] + [InlineData("terminate")] + public async Task Poll_EachPublishedTransitionNotifiesAllWaiters(string transition) + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + if (transition != "attach") + { + manager.OnWorkerAttached(1); + } + + if (transition is "assign" or "terminate") + { + manager.OnWorkerStartStream(1, "worker"); + } + + if (transition == "terminate") + { + manager.Assign(CreateAssignment()); + } + + long revision = manager.State.Revision; + Task[] polls = Enumerable.Range(0, 4) + .Select(_ => manager.WaitForChangeAsync(revision)).ToArray(); + Assert.Equal(4, manager.PendingWaiterCount); + Assert.All(polls, poll => Assert.False(poll.IsCompleted)); + + switch (transition) + { + case "attach": + manager.OnWorkerAttached(1); + break; + case "start": + manager.OnWorkerStartStream(1, "worker"); + break; + case "assign": + manager.Assign(CreateAssignment()); + break; + case "terminate": + manager.OnSessionTerminated(1); + break; + } + + WorkerStatePollResult[] results = await Task.WhenAll(polls).WaitAsync(TestTimeout); + Assert.All(results, result => + { + Assert.True(result.HasChanged); + Assert.Same(manager.State, result.State); + Assert.Equal(revision + 1, result.State!.Revision); + }); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task Poll_UnchangedStateReturnsNoChangeAtSixtySecondDeadline() + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + WorkerPodState initial = manager.State; + Task poll = manager.WaitForChangeAsync(0); + + Assert.False(poll.IsCompleted); + Assert.Equal(1, manager.PendingWaiterCount); + PollClock.ScheduledTimer timer = Assert.Single(clock.Timers); + Assert.Equal(TimeSpan.FromSeconds(60), timer.DueTime); + Assert.Equal(Timeout.InfiniteTimeSpan, timer.Period); + + timer.Fire(); + WorkerStatePollResult result = await poll.WaitAsync(TestTimeout); + + Assert.Same(WorkerStatePollResult.NoChange, result); + Assert.False(result.HasChanged); + Assert.Null(result.State); + Assert.Same(initial, manager.State); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task Poll_CancellationRemovesOnlyCanceledWaiter() + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + using CancellationTokenSource cancellation = new(); + Task canceledPoll = manager.WaitForChangeAsync(0, cancellation.Token); + Task activePoll = manager.WaitForChangeAsync(0); + Assert.Equal(2, manager.PendingWaiterCount); + + cancellation.Cancel(); + OperationCanceledException exception = await Assert.ThrowsAnyAsync( + () => canceledPoll.WaitAsync(TestTimeout)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, manager.PendingWaiterCount); + Assert.False(activePoll.IsCompleted); + Assert.Equal(0, manager.State.Revision); + + manager.OnWorkerAttached(1); + Assert.True((await activePoll.WaitAsync(TestTimeout)).HasChanged); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Fact] + public async Task Poll_TimeoutRemovesOnlyExpiredWaiter() + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + Task expiredPoll = manager.WaitForChangeAsync(0); + Task activePoll = manager.WaitForChangeAsync(0); + + clock.Timers[0].Fire(); + Assert.False((await expiredPoll.WaitAsync(TestTimeout)).HasChanged); + Assert.Equal(1, manager.PendingWaiterCount); + Assert.False(activePoll.IsCompleted); + + manager.OnWorkerAttached(1); + Assert.True((await activePoll.WaitAsync(TestTimeout)).HasChanged); + Assert.Equal(0, manager.PendingWaiterCount); + clock.VerifyTimersDisposed(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Poll_PreCanceledRequestDoesNotRegisterWaiter(bool staleRevision) + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + if (staleRevision) + { + manager.OnWorkerAttached(1); + } + + using CancellationTokenSource cancellation = new(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => manager.WaitForChangeAsync(0, cancellation.Token)); + + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Empty(clock.Timers); + } + + [Fact] + public async Task Poll_NoOpNotificationsAndAssignmentRetriesDoNotWakeWaiter() + { + WorkerPodStateManager manager = CreateReadyManager(); + manager.Assign(CreateAssignment()); + WorkerPodState state = manager.State; + using CancellationTokenSource cancellation = new(); + Task poll = manager.WaitForChangeAsync(state.Revision, cancellation.Token); + + Assert.False(manager.OnWorkerAttached(1)); + Assert.False(manager.OnWorkerStartStream(1, "worker")); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.AssignmentConflict, manager.Assign(CreateAssignment("other"))); + Assert.False(poll.IsCompleted); + Assert.Equal(1, manager.PendingWaiterCount); + Assert.Same(state, manager.State); + + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => poll.WaitAsync(TestTimeout)); + Assert.Equal(0, manager.PendingWaiterCount); + } + + [Fact] + public async Task Poll_RegistrationRacingPublicationCannotMissChange() + { + using CancellationTokenSource timeout = new(TestTimeout); + for (int iteration = 0; iteration < 32; iteration++) + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task poll = Task.Run(async () => + { + await start.Task; + return await manager.WaitForChangeAsync(0, timeout.Token); + }); + Task publication = Task.Run(async () => + { + await start.Task; + manager.OnWorkerAttached(1); + }); + + start.SetResult(); + await Task.WhenAll(poll, publication).WaitAsync(timeout.Token); + + Assert.Same(manager.State, (await poll).State); + Assert.Equal(0, manager.PendingWaiterCount); + } + } + + [Fact] + public async Task Poll_CancellationRacingPublicationAlwaysCleansUp() + { + using CancellationTokenSource timeout = new(TestTimeout); + for (int iteration = 0; iteration < 32; iteration++) + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + using CancellationTokenSource cancellation = new(); + Task poll = manager.WaitForChangeAsync(0, cancellation.Token); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task cancel = Task.Run(async () => + { + await start.Task; + cancellation.Cancel(); + }); + Task publish = Task.Run(async () => + { + await start.Task; + manager.OnWorkerAttached(1); + }); + + start.SetResult(); + await Task.WhenAll(cancel, publish).WaitAsync(timeout.Token); + try + { + Assert.Same(manager.State, (await poll.WaitAsync(timeout.Token)).State); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cancellation.Token) + { + // Either cancellation or publication may win, but neither may leave a registered waiter. + } + + Assert.Equal(0, manager.PendingWaiterCount); + } + } + + [Fact] + public async Task Poll_TimeoutRacingPublicationAlwaysCleansUp() + { + using CancellationTokenSource timeout = new(TestTimeout); + for (int iteration = 0; iteration < 32; iteration++) + { + PollClock clock = new(); + WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); + Task poll = manager.WaitForChangeAsync(0); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task expire = Task.Run(async () => + { + await start.Task; + clock.Timers[0].Fire(); + }); + Task publish = Task.Run(async () => + { + await start.Task; + manager.OnWorkerAttached(1); + }); + + start.SetResult(); + await Task.WhenAll(expire, publish).WaitAsync(timeout.Token); + WorkerStatePollResult result = await poll.WaitAsync(timeout.Token); + if (result.HasChanged) + { + Assert.Same(manager.State, result.State); + } + else + { + Assert.Same(WorkerStatePollResult.NoChange, result); + } + + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Same(manager.State, (await manager.WaitForChangeAsync(0, timeout.Token)).State); + clock.VerifyTimersDisposed(); + } + } + + [Fact] + public async Task Poll_TimerCreationFailureDoesNotLeakWaiter() + { + Mock timeProvider = new(); + timeProvider.Setup(provider => provider.CreateTimer( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new InvalidOperationException("Timer unavailable.")); + WorkerPodStateManager manager = new(CreateOptions(), timeProvider.Object); + + await Assert.ThrowsAsync(() => manager.WaitForChangeAsync(0)); + + Assert.Equal(0, manager.PendingWaiterCount); + Assert.Equal(0, manager.State.Revision); + } + + private sealed class PollClock + { + public PollClock() + { + Mock provider = new(); + provider.Setup(clock => clock.CreateTimer( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + { + ScheduledTimer timer = new(callback, state, dueTime, period); + Timers.Add(timer); + return timer.Timer.Object; + }); + Provider = provider.Object; + } + + public TimeProvider Provider { get; } + + public List Timers { get; } = []; + + public void VerifyTimersDisposed() + { + Assert.All(Timers, timer => timer.Timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce())); + } + + public sealed record ScheduledTimer(TimerCallback Callback, object? State, TimeSpan DueTime, TimeSpan Period) + { + public Mock Timer { get; } = new(); + + public void Fire() => Callback(State); + } + } +} diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs new file mode 100644 index 0000000000..8055cec280 --- /dev/null +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs @@ -0,0 +1,340 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.State; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Azure.Functions.WorkerProxy.Tests.State; + +public partial class WorkerPodStateManagerTests +{ + [Fact] + public void InitialState_IsUnassignedAtRevisionZero() + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + WorkerPodState state = manager.State; + + Assert.Equal("pod", state.PodName); + Assert.Equal(0, state.Revision); + Assert.Null(state.SessionId); + Assert.False(state.IsWorkerAttached); + Assert.False(state.IsWorkerReady); + Assert.Null(state.WorkerId); + Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); + Assert.Equal(WorkerPodStatus.None, state.PodStatus); + Assert.Null(state.FunctionAppName); + Assert.Null(state.FunctionGroupName); + Assert.Null(state.IsAlwaysReady); + Assert.Same(state, manager.State); + } + + [Fact] + public void Lifecycle_PublishesImmutableMonotonicSnapshots() + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + WorkerPodState initial = manager.State; + Assert.True(manager.OnWorkerAttached(1)); + WorkerPodState attached = manager.State; + Assert.False(attached.IsWorkerReady); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + WorkerPodState started = manager.State; + Assert.True(started.IsWorkerReady); + Assert.Equal(WorkerPodStatus.None, started.PodStatus); + + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + WorkerPodState assigned = manager.State; + Assert.Equal(WorkerAssignmentState.Ready, assigned.AssignmentState); + Assert.Equal(WorkerPodStatus.ReadyForRequest, assigned.PodStatus); + Assert.Equal("app", assigned.FunctionAppName); + Assert.Equal("http", assigned.FunctionGroupName); + Assert.False(assigned.IsAlwaysReady); + Assert.Equal("worker", assigned.WorkerId); + Assert.Equal(1, assigned.SessionId); + + Assert.True(manager.OnSessionTerminated(1)); + WorkerPodState failed = manager.State; + Assert.False(failed.IsWorkerReady); + Assert.False(failed.IsWorkerAttached); + Assert.Equal(WorkerAssignmentState.Failed, failed.AssignmentState); + Assert.Equal(WorkerPodStatus.None, failed.PodStatus); + Assert.Equal("app", failed.FunctionAppName); + Assert.Equal(new long[] { 0, 1, 2, 3, 4 }, + new[] { initial.Revision, attached.Revision, started.Revision, assigned.Revision, failed.Revision }); + + Assert.False(initial.IsWorkerAttached); + Assert.Null(attached.WorkerId); + Assert.Equal(WorkerAssignmentState.Unassigned, started.AssignmentState); + Assert.True(assigned.IsWorkerReady); + } + + [Fact] + public void NotReadyAssignment_DoesNotClaimIdentityOrChangeRevision() + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + WorkerPodState initial = manager.State; + Assert.Equal(WorkerAssignmentResult.WorkerNotReady, manager.Assign(CreateAssignment("rejected"))); + Assert.Same(initial, manager.State); + + manager.OnWorkerAttached(1); + WorkerPodState attached = manager.State; + Assert.Equal(WorkerAssignmentResult.WorkerNotReady, manager.Assign(CreateAssignment("also-rejected"))); + Assert.Same(attached, manager.State); + + manager.OnWorkerStartStream(1, "worker"); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment("accepted"))); + Assert.Equal("accepted", manager.State.FunctionAppName); + } + + [Fact] + public void AssignmentReplayAndConflict_DoNotChangeRevision() + { + WorkerPodStateManager manager = CreateReadyManager(); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + WorkerPodState state = manager.State; + + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.AssignmentConflict, manager.Assign(CreateAssignment("other"))); + Assert.Same(state, manager.State); + } + + [Fact] + public void TerminalFailure_IsStickyAndConflictingIdentityStillConflicts() + { + WorkerPodStateManager manager = CreateReadyManager(); + manager.Assign(CreateAssignment()); + manager.OnSessionTerminated(1); + WorkerPodState failed = manager.State; + + Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.AssignmentConflict, manager.Assign(CreateAssignment("other"))); + Assert.False(manager.OnWorkerAttached(2)); + Assert.False(manager.OnWorkerStartStream(2, "replacement")); + Assert.False(manager.OnWorkerStartStream(1, "worker")); + Assert.False(manager.OnSessionTerminated(1)); + Assert.Same(failed, manager.State); + } + + [Fact] + public void UnassignedSession_CanBeReplacedWithoutResettingRevision() + { + WorkerPodStateManager manager = CreateReadyManager(); + manager.OnSessionTerminated(1); + Assert.Equal(WorkerAssignmentState.Unassigned, manager.State.AssignmentState); + Assert.Equal(WorkerAssignmentResult.WorkerNotReady, manager.Assign(CreateAssignment())); + + Assert.True(manager.OnWorkerAttached(2)); + Assert.False(manager.State.IsWorkerReady); + Assert.True(manager.OnWorkerStartStream(2, "replacement")); + WorkerPodState replacement = manager.State; + + Assert.False(manager.OnWorkerAttached(1)); + Assert.False(manager.OnWorkerStartStream(1, "old-worker")); + Assert.False(manager.OnSessionTerminated(1)); + Assert.Same(replacement, manager.State); + Assert.Equal(5, replacement.Revision); + Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(6, manager.State.Revision); + } + + [Fact] + public void RuntimeOnlyTermination_PreventsLateWorkerAttachmentForThatSession() + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + WorkerPodState initial = manager.State; + + Assert.False(manager.OnSessionTerminated(1)); + Assert.False(manager.OnWorkerAttached(1)); + Assert.False(manager.OnWorkerStartStream(1, "late-worker")); + Assert.Same(initial, manager.State); + Assert.True(manager.OnWorkerAttached(2)); + } + + [Fact] + public void DuplicateLifecycleNotifications_DoNotChangeRevision() + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState ready = manager.State; + + Assert.False(manager.OnWorkerAttached(1)); + Assert.False(manager.OnWorkerStartStream(1, "worker")); + Assert.Same(ready, manager.State); + } + + [Fact] + public void InconsistentLifecycleNotifications_AreRejectedWithoutMutation() + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState ready = manager.State; + + Assert.Throws(() => manager.OnWorkerAttached(2)); + Assert.Throws(() => manager.OnSessionTerminated(2)); + Assert.Throws(() => manager.OnWorkerStartStream(1, "different-worker")); + Assert.Same(ready, manager.State); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void InvalidWorkerIdentity_DoesNotEnableReadiness(string? workerId) + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + manager.OnWorkerAttached(1); + WorkerPodState attached = manager.State; + + Assert.ThrowsAny(() => manager.OnWorkerStartStream(1, workerId!)); + Assert.Same(attached, manager.State); + Assert.False(manager.State.IsWorkerReady); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void InvalidSessionIdentity_IsRejected(long sessionId) + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + Assert.Throws(() => manager.OnWorkerAttached(sessionId)); + Assert.Throws(() => manager.OnWorkerStartStream(sessionId, "worker")); + Assert.Throws(() => manager.OnSessionTerminated(sessionId)); + Assert.Equal(0, manager.State.Revision); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void MissingPodIdentity_IsRejected(string? podName) + { + Assert.ThrowsAny(() => new WorkerPodStateManager(CreateOptions(podName!), TimeProvider.System)); + } + + [Fact] + public void NullTimeProvider_IsRejected() + { + Assert.Throws(() => new WorkerPodStateManager(CreateOptions(), null!)); + } + + [Fact] + public void NullOptions_IsRejected() + { + Assert.Throws(() => new WorkerPodStateManager(null!, TimeProvider.System)); + } + + [Fact] + public void PodIdentity_IsCapturedDuringConstruction() + { + WorkerProxyOptions options = new() { PodName = "original-pod" }; + WorkerPodStateManager manager = new(Options.Create(options), TimeProvider.System); + + options.PodName = "different-pod"; + manager.OnWorkerAttached(1); + + Assert.Equal("original-pod", manager.State.PodName); + } + + [Fact] + public void NullAssignment_IsRejectedWithoutMutation() + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState ready = manager.State; + Assert.Throws(() => manager.Assign(null!)); + Assert.Same(ready, manager.State); + } + + [Fact] + public async Task ConcurrentEquivalentAssignments_PublishOnlyOnce() + { + WorkerPodStateManager manager = CreateReadyManager(); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task[] attempts = Enumerable.Range(0, 32).Select(_ => Task.Run(async () => + { + await start.Task; + return manager.Assign(CreateAssignment()); + })).ToArray(); + + start.SetResult(); + WorkerAssignmentResult[] results = await Task.WhenAll(attempts); + + Assert.All(results, result => Assert.Equal(WorkerAssignmentResult.Success, result)); + Assert.Equal(3, manager.State.Revision); + } + + [Fact] + public async Task ConcurrentConflictingAssignments_HaveOneWinningIdentity() + { + WorkerPodStateManager manager = CreateReadyManager(); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task<(string AppName, WorkerAssignmentResult Result)>[] attempts = Enumerable.Range(0, 32).Select(index => Task.Run(async () => + { + await start.Task; + string appName = $"app-{index}"; + return (appName, manager.Assign(CreateAssignment(appName))); + })).ToArray(); + + start.SetResult(); + (string AppName, WorkerAssignmentResult Result)[] results = await Task.WhenAll(attempts); + (string AppName, WorkerAssignmentResult Result) winner = Assert.Single(results, result => result.Result == WorkerAssignmentResult.Success); + + Assert.Equal(winner.AppName, manager.State.FunctionAppName); + Assert.Equal(31, results.Count(result => result.Result == WorkerAssignmentResult.AssignmentConflict)); + Assert.Equal(3, manager.State.Revision); + } + + [Fact] + public async Task AssignmentRacingTermination_CannotLeaveReadyState() + { + for (int iteration = 0; iteration < 32; iteration++) + { + WorkerPodStateManager manager = CreateReadyManager(); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task assignment = Task.Run(async () => + { + await start.Task; + return manager.Assign(CreateAssignment()); + }); + Task termination = Task.Run(async () => + { + await start.Task; + manager.OnSessionTerminated(1); + }); + + start.SetResult(); + await Task.WhenAll(assignment, termination); + + WorkerAssignmentResult result = await assignment; + Assert.False(manager.State.IsWorkerReady); + Assert.Equal(WorkerPodStatus.None, manager.State.PodStatus); + if (result == WorkerAssignmentResult.Success) + { + Assert.Equal(WorkerAssignmentState.Failed, manager.State.AssignmentState); + Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(CreateAssignment())); + Assert.Equal(4, manager.State.Revision); + } + else + { + Assert.Equal(WorkerAssignmentResult.WorkerNotReady, result); + Assert.Equal(WorkerAssignmentState.Unassigned, manager.State.AssignmentState); + Assert.Equal(3, manager.State.Revision); + } + } + } + + private static WorkerPodStateManager CreateReadyManager() + { + WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + return manager; + } + + private static WorkerAssignment CreateAssignment(string appName = "app") + => new(appName, "http", false, new Dictionary { ["SETTING"] = "value" }, "/home/site/wwwroot"); + + private static IOptions CreateOptions(string podName = "pod") + => Options.Create(new WorkerProxyOptions { PodName = podName }); +} diff --git a/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs b/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs index e75ec8393b..6de71a0cba 100644 --- a/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs +++ b/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs @@ -2,16 +2,87 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Http; +using Azure.Functions.WorkerProxy.Rpc; +using Azure.Functions.WorkerProxy.State; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Moq; using Xunit; namespace Azure.Functions.WorkerProxy.Tests; public class WorkerProxyApplicationTests { + [Fact] + public async Task CapabilityFinalizer_ResolvesHttpProviderAsSingleton() + { + await using WorkerProxyWebApplicationFactory factory = new(); + IWorkerCapabilityFinalizer finalizer = factory.Services.GetRequiredService(); + using IServiceScope scope = factory.Services.CreateScope(); + + Assert.IsType(finalizer); + Assert.Same(finalizer, scope.ServiceProvider.GetRequiredService()); + } + + [Fact] + public async Task StateManager_ResolvesAsSingletonWithConfiguredIdentity() + { + Dictionary configuration = new() { ["WorkerProxy:PodName"] = "configured-worker-pod" }; + await using WorkerProxyWebApplicationFactory factory = new(configuration); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using IServiceScope scope = factory.Services.CreateScope(); + + Assert.Same(TimeProvider.System, factory.Services.GetRequiredService()); + Assert.Same(manager, scope.ServiceProvider.GetRequiredService()); + Assert.Equal("configured-worker-pod", manager.State.PodName); + Assert.Equal(0, manager.State.Revision); + } + + [Fact] + public async Task StateManager_UsesTimeProviderFromContainer() + { + Mock provider = new(); + Mock timer = new(); + provider.Setup(clock => clock.CreateTimer( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(timer.Object); + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.Replace(ServiceDescriptor.Singleton(provider.Object))); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + + Task poll = manager.WaitForChangeAsync(0, timeout.Token); + + Assert.False(poll.IsCompleted); + provider.Verify(clock => clock.CreateTimer( + It.IsAny(), It.IsAny(), TimeSpan.FromSeconds(60), Timeout.InfiniteTimeSpan), Times.Once()); + manager.OnWorkerAttached(1); + Assert.True((await poll).HasChanged); + timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce()); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task Startup_RejectsMissingOrBlankPodName(string? podName) + { + Dictionary configuration = new() { ["WorkerProxy:PodName"] = podName }; + await using WorkerProxyWebApplicationFactory factory = new(configuration); + + OptionsValidationException exception = Assert.Throws( + () => factory.Services.GetRequiredService()); + + Assert.Contains(exception.Failures, failure => failure.Contains(nameof(WorkerProxyOptions.PodName), StringComparison.Ordinal)); + } + [Fact] public async Task ManagementListener_ProtectsAdminRoutes() { @@ -26,7 +97,7 @@ public async Task ManagementListener_ProtectsAdminRoutes() using HttpResponseMessage unsupportedMethodResponse = await client.PostAsync("/admin/instance/ready", content: null, timeout.Token); Assert.Equal(HttpStatusCode.MethodNotAllowed, unsupportedMethodResponse.StatusCode); - using HttpResponseMessage unrelatedRouteResponse = await client.GetAsync("/admin/worker/ready", timeout.Token); + using HttpResponseMessage unrelatedRouteResponse = await client.GetAsync("/admin/worker/unknown", timeout.Token); Assert.Equal(HttpStatusCode.NotFound, unrelatedRouteResponse.StatusCode); using HttpClient forwardingClient = factory.CreateHttpForwardingClient(); diff --git a/test/Functions.WorkerProxy.Tests/WorkerProxyEnvironmentConfigurationTests.cs b/test/Functions.WorkerProxy.Tests/WorkerProxyEnvironmentConfigurationTests.cs index a09a898e97..528ba09dee 100644 --- a/test/Functions.WorkerProxy.Tests/WorkerProxyEnvironmentConfigurationTests.cs +++ b/test/Functions.WorkerProxy.Tests/WorkerProxyEnvironmentConfigurationTests.cs @@ -8,6 +8,7 @@ using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.State; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -33,6 +34,7 @@ public async Task ProductionConfiguration_UsesWorkerProxyEndpointOptions() using EnvironmentVariableScope runtimeGrpcPort = new("WORKERPROXY__RUNTIMEGRPCPORT", "0"); using EnvironmentVariableScope workerGrpcPort = new("WORKERPROXY__WORKERGRPCPORT", "0"); using EnvironmentVariableScope httpPort = new("WORKERPROXY__HTTPPORT", "0"); + using EnvironmentVariableScope podName = new("WORKERPROXY__PODNAME", "environment-worker-pod"); using EnvironmentVariableScope urls = new("ASPNETCORE_URLS", $"http://127.0.0.1:{ambientPort}"); using EnvironmentVariableScope dotnetSetting = new("DOTNET_WORKER_PROXY_TEST_SETTING", "preserved"); await using WebApplication app = WorkerProxyApplication.Build([]); @@ -44,6 +46,7 @@ public async Task ProductionConfiguration_UsesWorkerProxyEndpointOptions() Assert.Equal(managementPort, address.Port); Assert.NotEqual(ambientPort, address.Port); Assert.Equal("preserved", app.Configuration["DOTNET_WORKER_PROXY_TEST_SETTING"]); + Assert.Equal("environment-worker-pod", app.Services.GetRequiredService().State.PodName); using HttpClient client = new() { BaseAddress = address }; using HttpResponseMessage response = await client.GetAsync("/admin/instance/ready", timeout.Token); Assert.Equal(HttpStatusCode.OK, response.StatusCode); diff --git a/test/Functions.WorkerProxy.Tests/WorkerProxyOptionsTests.cs b/test/Functions.WorkerProxy.Tests/WorkerProxyOptionsTests.cs index 363749aaa7..ad8af2e40b 100644 --- a/test/Functions.WorkerProxy.Tests/WorkerProxyOptionsTests.cs +++ b/test/Functions.WorkerProxy.Tests/WorkerProxyOptionsTests.cs @@ -28,6 +28,7 @@ public void Options_UseStableDefaults() public void Options_BindFromStandardConfiguration() { WorkerProxyOptions options = GetOptions( + "--WorkerProxy:PodName", "configured-worker-pod", "--WorkerProxy:ManagementPort", "41000", "--WorkerProxy:RuntimeGrpcPort", "41001", "--WorkerProxy:WorkerGrpcPort", "41002", @@ -35,6 +36,7 @@ public void Options_BindFromStandardConfiguration() "--WorkerProxy:WorkerHttpEndpoint", "http://localhost:41004", "--WorkerProxy:HttpProxyEndpoint", "https://worker-pod.example:48801"); + Assert.Equal("configured-worker-pod", options.PodName); Assert.Equal(41000, options.ManagementPort); Assert.Equal(41001, options.RuntimeGrpcPort); Assert.Equal(41002, options.WorkerGrpcPort); @@ -125,7 +127,7 @@ public void Options_AllowMultipleEphemeralPorts() private static WorkerProxyOptions GetOptions(params string[] args) { - using WebApplication app = WorkerProxyApplication.Build(args); + using WebApplication app = WorkerProxyApplication.Build(["--WorkerProxy:PodName", "test-worker-pod", .. args]); return app.Services.GetRequiredService>().Value; } diff --git a/test/Functions.WorkerProxy.Tests/WorkerProxyWebApplicationFactory.cs b/test/Functions.WorkerProxy.Tests/WorkerProxyWebApplicationFactory.cs index 670396f005..29f452b668 100644 --- a/test/Functions.WorkerProxy.Tests/WorkerProxyWebApplicationFactory.cs +++ b/test/Functions.WorkerProxy.Tests/WorkerProxyWebApplicationFactory.cs @@ -22,6 +22,7 @@ public WorkerProxyWebApplicationFactory( IReadOnlyDictionary? configurationValues = null, Action? configureServices = null) { Dictionary values = configurationValues is null ? [] : new Dictionary(configurationValues); + values.TryAdd($"{WorkerProxyOptions.SectionName}:{nameof(WorkerProxyOptions.PodName)}", "test-worker-pod"); values.TryAdd($"{WorkerProxyOptions.SectionName}:{nameof(WorkerProxyOptions.ManagementPort)}", "0"); values.TryAdd($"{WorkerProxyOptions.SectionName}:{nameof(WorkerProxyOptions.RuntimeGrpcPort)}", "0"); values.TryAdd($"{WorkerProxyOptions.SectionName}:{nameof(WorkerProxyOptions.WorkerGrpcPort)}", "0"); From 74b3e37d43f864896586051fb02b7d5e2eed1f93 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 12:17:18 -0700 Subject: [PATCH 02/10] Fix WorkerProxy development startup and review feedback Supply a development pod name in the launch profile and use explicit ordinal string comparisons in the affected tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Properties/launchSettings.json | 1 + .../Management/ManagementApiEndpointTests.cs | 9 +++++---- .../Management/ManagementApiHandlersTests.cs | 4 ++-- .../State/WorkerAssignmentTests.cs | 13 +++++++------ .../State/WorkerPodStateManagerTests.Polling.cs | 4 ++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/Functions.WorkerProxy/Properties/launchSettings.json b/src/Functions.WorkerProxy/Properties/launchSettings.json index 8817fc3b1a..3930a306ea 100644 --- a/src/Functions.WorkerProxy/Properties/launchSettings.json +++ b/src/Functions.WorkerProxy/Properties/launchSettings.json @@ -6,6 +6,7 @@ "launchBrowser": false, "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", + "WORKERPROXY__PODNAME": "local-worker-pod", "WORKERPROXY__MANAGEMENTPORT": "8080", "WORKERPROXY__HTTPPORT": "28080", "WORKERPROXY__HTTPPROXYENDPOINT": "http://localhost:28080", diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs index da3eacc14f..f113d996c4 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs @@ -105,7 +105,8 @@ public async Task InvalidAssignmentDoesNotClaimIdentity(string field, string? va using StringContent content = JsonBody(body.ToJsonString()); using HttpResponseMessage response = await client.PostAsync(AssignPath, content, timeout.Token); - await AssertValidationAsync(response, timeout.Token, (code, code == "InvalidBody" ? "request" : field)); + await AssertValidationAsync(response, timeout.Token, + (code, string.Equals(code, "InvalidBody", StringComparison.Ordinal) ? "request" : field)); WorkerPodState state = factory.Services.GetRequiredService().State; Assert.Equal(0, state.Revision); Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); @@ -131,7 +132,7 @@ await AssertValidationAsync(response, timeout.Token, ("Required", "functionGroupName"), ("Required", "isAlwaysReady"), ("Required", "functionAppDirectory"), - (body == "{}" ? "Required" : "InvalidValue", "environment")); + (string.Equals(body, "{}", StringComparison.Ordinal) ? "Required" : "InvalidValue", "environment")); string json = await response.Content.ReadAsStringAsync(timeout.Token); Assert.DoesNotContain("PRIVATE_SETTING", json); Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); @@ -182,7 +183,7 @@ public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string pa { Version = HttpVersion.Version20, VersionPolicy = HttpVersionPolicy.RequestVersionExact, - Content = method == "POST" ? JsonBody("{}") : null + Content = string.Equals(method, "POST", StringComparison.Ordinal) ? JsonBody("{}") : null }; using HttpResponseMessage response = await rpcClient.SendAsync(request, timeout.Token); Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); @@ -191,7 +192,7 @@ public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string pa using HttpClient forwardingClient = factory.CreateHttpForwardingClient(); using HttpRequestMessage forwardingRequest = new(new HttpMethod(method), path) { - Content = method == "POST" ? JsonBody("{}") : null + Content = string.Equals(method, "POST", StringComparison.Ordinal) ? JsonBody("{}") : null }; using HttpResponseMessage forwardingResponse = await forwardingClient.SendAsync(forwardingRequest, timeout.Token); Assert.Equal(HttpStatusCode.NotFound, forwardingResponse.StatusCode); diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs index 41189f0031..92833fb7b4 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs @@ -380,7 +380,7 @@ public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapsho { PollClock clock = new(); WorkerPodStateManager manager = CreateManager(clock.Provider); - if (transition != "attach") + if (!string.Equals(transition, "attach", StringComparison.Ordinal)) { manager.OnWorkerAttached(1); } @@ -390,7 +390,7 @@ public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapsho manager.OnWorkerStartStream(1, "worker"); } - if (transition == "terminate") + if (string.Equals(transition, "terminate", StringComparison.Ordinal)) { Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); } diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs index 0823821d9b..e48364b8d6 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs @@ -39,19 +39,20 @@ public void Equality_UsesEveryFieldAndOrdinalStrings(string changedField) WorkerAssignment original = Create(new Dictionary { ["KEY"] = "Value" }); Dictionary environment = new(StringComparer.OrdinalIgnoreCase) { - [changedField == "key" ? "key" : "KEY"] = changedField == "value" ? "value" : "Value" + [string.Equals(changedField, "key", StringComparison.Ordinal) ? "key" : "KEY"] = + string.Equals(changedField, "value", StringComparison.Ordinal) ? "value" : "Value" }; - if (changedField == "count") + if (string.Equals(changedField, "count", StringComparison.Ordinal)) { environment.Add("EXTRA", "Value"); } WorkerAssignment changed = new( - changedField == "app" ? "APP" : "app", - changedField == "group" ? "HTTP" : "http", - changedField == "alwaysReady", + string.Equals(changedField, "app", StringComparison.Ordinal) ? "APP" : "app", + string.Equals(changedField, "group", StringComparison.Ordinal) ? "HTTP" : "http", + string.Equals(changedField, "alwaysReady", StringComparison.Ordinal), environment, - changedField == "directory" ? "/HOME/site/wwwroot" : "/home/site/wwwroot"); + string.Equals(changedField, "directory", StringComparison.Ordinal) ? "/HOME/site/wwwroot" : "/home/site/wwwroot"); Assert.False(original.IsEquivalentTo(changed)); Assert.False(changed.IsEquivalentTo(original)); diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs index 71c9a2a89d..e6b0175faa 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs @@ -63,7 +63,7 @@ public async Task Poll_EachPublishedTransitionNotifiesAllWaiters(string transiti { PollClock clock = new(); WorkerPodStateManager manager = new(CreateOptions(), clock.Provider); - if (transition != "attach") + if (!string.Equals(transition, "attach", StringComparison.Ordinal)) { manager.OnWorkerAttached(1); } @@ -73,7 +73,7 @@ public async Task Poll_EachPublishedTransitionNotifiesAllWaiters(string transiti manager.OnWorkerStartStream(1, "worker"); } - if (transition == "terminate") + if (string.Equals(transition, "terminate", StringComparison.Ordinal)) { manager.Assign(CreateAssignment()); } From 08a46ea0af62cbadae6b5d63b167c0177e86662e Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 12:55:53 -0700 Subject: [PATCH 03/10] Synchronize delayed worker read processing in readiness test Pump the captured test synchronization context until the delayed inbound continuation finishes before asserting replacement readiness. Preserve nonblocking production termination behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Rpc/FunctionRpcRelayTests.Readiness.cs | 65 ++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs index ae46146dd5..410e823d2a 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs @@ -292,25 +292,37 @@ public async Task Relay_DelayedStartStreamCannotChangeReplacementReadiness() await using FunctionRpcRelay relay = CreateInProcessRelay(manager); using CancellationTokenSource timeout = new(TestTimeout); using CancellationTokenSource runtimeCancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); - TaskCompletionSource delayedRead = new(); + TaskCompletionSource delayedRead = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource currentRead = new(TaskCreationOptions.RunContinuationsAsynchronously); + PumpingSynchronizationContext readContext = new(); Mock> oldReader = new(MockBehavior.Strict); oldReader.Setup(reader => reader.MoveNext(It.IsAny())).Returns(delayedRead.Task); - oldReader.SetupGet(reader => reader.Current).Returns(new StreamingMessage + oldReader.SetupGet(reader => reader.Current).Returns(() => { - StartStream = new() { WorkerId = "old-worker" } + currentRead.TrySetResult(true); + return new StreamingMessage { StartStream = new() { WorkerId = "old-worker" } }; }); - // Register the read without a test synchronization context. Completing delayedRead then processes it inline. - Task oldWorker = await Task.Factory.StartNew( - () => relay.AttachAsync(FunctionRpcRelaySide.Worker, oldReader.Object, new TestServerStreamWriter(), timeout.Token), - timeout.Token, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); + Task oldWorker; + SynchronizationContext? previousContext = SynchronizationContext.Current; + try + { + SynchronizationContext.SetSynchronizationContext(readContext); + oldWorker = relay.AttachAsync(FunctionRpcRelaySide.Worker, + oldReader.Object, new TestServerStreamWriter(), timeout.Token); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + Task oldRuntime = relay.AttachAsync(FunctionRpcRelaySide.Runtime, new BlockingStreamReader(), new TestServerStreamWriter(), runtimeCancellation.Token); try { runtimeCancellation.Cancel(); - await Task.WhenAll(oldRuntime, oldWorker).WaitAsync(timeout.Token); + await readContext.RunUntilAsync(Task.WhenAll(oldRuntime, oldWorker), timeout.Token); Assert.False(manager.State.IsWorkerReady); Task replacement = relay.AttachAsync(FunctionRpcRelaySide.Worker, @@ -319,6 +331,9 @@ public async Task Relay_DelayedStartStreamCannotChangeReplacementReadiness() Assert.True(ready.IsWorkerReady); delayedRead.SetResult(true); + // Current only marks entry. The pump returns after the entire callback, including + // ProcessInboundMessage and the stale reader's return, has finished. + await readContext.RunUntilAsync(currentRead.Task, timeout.Token); oldReader.VerifyGet(reader => reader.Current, Times.Once); Assert.Same(ready, manager.State); Assert.Equal("test-worker", manager.State.WorkerId); @@ -335,4 +350,38 @@ public async Task Relay_DelayedStartStreamCannotChangeReplacementReadiness() private static WorkerAssignment CreateWorkerAssignment() => new("test-app", "test-group", isAlwaysReady: false, environment: new Dictionary(), functionAppDirectory: "/home/site/wwwroot"); + + private sealed class PumpingSynchronizationContext : SynchronizationContext + { + private readonly Channel<(SendOrPostCallback Callback, object? State)> _callbacks = + Channel.CreateUnbounded<(SendOrPostCallback, object?)>(); + + public override void Post(SendOrPostCallback callback, object? state) => + _callbacks.Writer.TryWrite((callback, state)); + + public async Task RunUntilAsync(Task completion, CancellationToken cancellationToken) + { + // Wake a waiting pump even when completion happens outside a queued callback. + _ = completion.ContinueWith( + _ => Post(static _ => { }, null), + CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + while (!completion.IsCompleted) + { + (SendOrPostCallback callback, object? state) = await _callbacks.Reader.ReadAsync(cancellationToken); + SynchronizationContext? previousContext = Current; + try + { + SetSynchronizationContext(this); + callback(state); + } + finally + { + SetSynchronizationContext(previousContext); + } + } + + await completion.WaitAsync(cancellationToken); + } + } } From 04f70cdfdef855eeb28e0e119d3ed0ec7a26ec85 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 13:28:10 -0700 Subject: [PATCH 04/10] Document explicit management API JSON parsing rationale Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Management/ManagementApiEndpoints.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs index 5a67d6f222..84c840ae04 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -13,6 +13,12 @@ namespace Azure.Functions.WorkerProxy.Management; /// /// Registers worker lifecycle APIs on the management listener. /// +/// +/// POST handlers explicitly use ReadFromJsonAsync rather than automatic body binding so malformed JSON, +/// incompatible field types, and unsupported content types return our Host-aligned HTTP 400 InvalidBody +/// validation envelope. Automatic binding can reject requests before the handler runs with framework-owned +/// 400/415 responses that do not guarantee that envelope. +/// internal static class ManagementApiEndpoints { public static void Map(IEndpointRouteBuilder endpoints) From 49e6d5cde507cddbfae6f42582737e007cee40ec Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 15:39:59 -0700 Subject: [PATCH 05/10] Align WorkerProxy management APIs with resource-oriented contract Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Management/InstanceStatePollRequest.cs | 12 - .../Management/ManagementApiEndpoints.cs | 43 ++- .../Management/ManagementApiHandlers.cs | 19 +- .../Management/WorkerApiError.cs | 4 +- .../Management/WorkerProxyJsonContext.cs | 1 - .../State/WorkerAssignmentResult.cs | 3 +- .../State/WorkerPodStateManager.cs | 4 +- .../Management/ManagementApiEndpointTests.cs | 339 +++++++++++++++--- .../Management/ManagementApiHandlersTests.cs | 67 ++-- .../Management/WorkerProxyJsonContextTests.cs | 45 +-- ...ionRpcRelayTests.CapabilityFinalization.cs | 4 +- .../Rpc/FunctionRpcRelayTests.Management.cs | 70 ++-- .../Rpc/FunctionRpcRelayTests.Readiness.cs | 8 +- .../Rpc/FunctionRpcRelayTests.cs | 2 +- .../WorkerPodStateManagerTests.Polling.cs | 2 +- .../State/WorkerPodStateManagerTests.cs | 21 +- 16 files changed, 423 insertions(+), 221 deletions(-) delete mode 100644 src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs diff --git a/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs b/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs deleted file mode 100644 index adb0fea557..0000000000 --- a/src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -namespace Azure.Functions.WorkerProxy.Management; - -/// -/// Requests a current snapshot when the revision is omitted or null, or waits for a newer revision when supplied. -/// -internal sealed class InstanceStatePollRequest -{ - public long? LastKnownRevision { get; init; } -} diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs index 84c840ae04..78834ee783 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; +using System.Globalization; using System.Text.Json; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.State; @@ -14,7 +16,7 @@ namespace Azure.Functions.WorkerProxy.Management; /// Registers worker lifecycle APIs on the management listener. /// /// -/// POST handlers explicitly use ReadFromJsonAsync rather than automatic body binding so malformed JSON, +/// The PUT handler explicitly uses ReadFromJsonAsync rather than automatic body binding so malformed JSON, /// incompatible field types, and unsupported content types return our Host-aligned HTTP 400 InvalidBody /// validation envelope. Automatic binding can reject requests before the handler runs with framework-owned /// 400/415 responses that do not guarantee that envelope. @@ -23,9 +25,17 @@ internal static class ManagementApiEndpoints { public static void Map(IEndpointRouteBuilder endpoints) { - endpoints.MapGet("/admin/worker/ready", ManagementApiHandlers.GetWorkerReady).AllowAnonymous(); - endpoints.MapPost("/admin/worker/assign", AssignWorkerAsync).AllowAnonymous(); - endpoints.MapPost("/admin/infra/instanceState", GetInstanceStateAsync).AllowAnonymous(); + endpoints.MapGet("/admin/worker/ready", ManagementApiHandlers.GetWorkerReady) + .AddEndpointFilter(DisableCaching).AllowAnonymous(); + endpoints.MapPut("/admin/worker/assignment", AssignWorkerAsync).AllowAnonymous(); + endpoints.MapGet("/admin/worker/state", GetInstanceStateAsync) + .AddEndpointFilter(DisableCaching).AllowAnonymous(); + } + + private static ValueTask DisableCaching(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + { + context.HttpContext.Response.Headers.CacheControl = "no-store"; + return next(context); } private static async Task AssignWorkerAsync(HttpRequest request, WorkerPodStateManager manager) @@ -51,22 +61,21 @@ private static async Task AssignWorkerAsync(HttpRequest request, Worker private static async Task GetInstanceStateAsync(HttpRequest request, WorkerPodStateManager manager) { - if (!request.HasJsonContentType()) + long? lastKnownRevision = null; + if (request.Query.TryGetValue("lastKnownRevision", out var revisions)) { - return ManagementApiHandlers.InvalidBody(); - } + string? value = revisions.Count == 1 ? revisions[0] : null; + // Accept invariant ASCII digits with an optional leading sign, not whitespace or JSON syntax. + if (value is null + || value.AsSpan().IndexOfAnyExcept("+-0123456789") >= 0 + || !long.TryParse(value, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out long revision)) + { + return ManagementApiHandlers.InvalidRevision(); + } - InstanceStatePollRequest? poll; - try - { - poll = await request.ReadFromJsonAsync( - WorkerProxyJsonContext.Default.InstanceStatePollRequest, request.HttpContext.RequestAborted); - } - catch (JsonException) - { - return ManagementApiHandlers.InvalidBody(); + lastKnownRevision = revision; } - return await ManagementApiHandlers.GetInstanceStateAsync(poll, manager, request.HttpContext.RequestAborted); + return await ManagementApiHandlers.GetInstanceStateAsync(lastKnownRevision, manager, request.HttpContext.RequestAborted); } } diff --git a/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs index 869f8db7a0..fb9a256cbe 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs @@ -83,29 +83,25 @@ public static IResult AssignWorker(WorkerAssignRequest? request, WorkerPodStateM functionAppDirectory); return manager.Assign(assignment) switch { - WorkerAssignmentResult.Success => TypedResults.Ok(), + WorkerAssignmentResult.Created => TypedResults.Created("/admin/worker/assignment"), + WorkerAssignmentResult.AlreadyAssigned => TypedResults.NoContent(), WorkerAssignmentResult.WorkerNotReady => Error( StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerNotReady, "The worker has not established a valid StartStream."), WorkerAssignmentResult.AssignmentConflict => Error( StatusCodes.Status409Conflict, WorkerApiErrorCodes.AssignmentConflict, "The pod is already assigned to a different assignment."), WorkerAssignmentResult.WorkerTerminated => Error( - StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerTerminated, "The assigned worker stream has terminated."), + StatusCodes.Status409Conflict, WorkerApiErrorCodes.WorkerTerminated, "The assigned worker stream has terminated."), _ => throw new InvalidOperationException("Unexpected worker assignment result.") }; } public static async Task GetInstanceStateAsync( - InstanceStatePollRequest? request, + long? revision, WorkerPodStateManager manager, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (request is null) - { - return InvalidBody(); - } - - if (request.LastKnownRevision is not { } lastKnownRevision) + if (revision is not { } lastKnownRevision) { return StateResponse(manager.State); } @@ -113,7 +109,7 @@ public static async Task GetInstanceStateAsync( // Revisions never decrease, so a revision valid here remains valid when the manager registers the poll. if (lastKnownRevision < 0 || lastKnownRevision > manager.State.Revision) { - return ValidationError([new(WorkerApiErrorCodes.InvalidRevision, "lastKnownRevision")]); + return InvalidRevision(); } WorkerStatePollResult result = await manager.WaitForChangeAsync(lastKnownRevision, cancellationToken); @@ -123,6 +119,9 @@ public static async Task GetInstanceStateAsync( internal static IResult InvalidBody() => ValidationError([new(WorkerApiErrorCodes.InvalidBody, "request")]); + internal static IResult InvalidRevision() => + ValidationError([new(WorkerApiErrorCodes.InvalidRevision, "lastKnownRevision")]); + private static IResult ValidationError(IReadOnlyList errors) => TypedResults.Json(new RequestValidationResponse(errors), WorkerProxyJsonContext.Default.RequestValidationResponse, statusCode: StatusCodes.Status400BadRequest); diff --git a/src/Functions.WorkerProxy/Management/WorkerApiError.cs b/src/Functions.WorkerProxy/Management/WorkerApiError.cs index 6b8dedcb41..5e21d5927c 100644 --- a/src/Functions.WorkerProxy/Management/WorkerApiError.cs +++ b/src/Functions.WorkerProxy/Management/WorkerApiError.cs @@ -9,9 +9,9 @@ namespace Azure.Functions.WorkerProxy.Management; /// A case-sensitive contract identifier; existing codes must not be renamed or repurposed. /// Diagnostic text that may change and must not be used for client decisions. /// -/// WorkerNotReady (503) permits retry after readiness. WorkerTerminated (503) is terminal for the +/// WorkerNotReady (503) permits retry after readiness. WorkerTerminated (409) is terminal for the /// assigned session; retrying the same assignment on this pod cannot recover it. /// AssignmentConflict (409) rejects a different assignment; do not retry that request unchanged. -/// Clients must inspect Code to distinguish the two 503 outcomes and handle unknown codes gracefully. +/// Clients must inspect Code to distinguish the two 409 outcomes and handle unknown codes gracefully. /// internal sealed record WorkerApiError(string Code, string? Detail = null); diff --git a/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs b/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs index b22ee1d790..f67caec199 100644 --- a/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs +++ b/src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs @@ -10,7 +10,6 @@ namespace Azure.Functions.WorkerProxy.Management; PropertyNameCaseInsensitive = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(WorkerAssignRequest))] -[JsonSerializable(typeof(InstanceStatePollRequest))] [JsonSerializable(typeof(WorkerInstanceState))] [JsonSerializable(typeof(WorkerApiErrorResponse))] [JsonSerializable(typeof(RequestValidationResponse))] diff --git a/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs b/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs index e951cd6113..81053169a6 100644 --- a/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs +++ b/src/Functions.WorkerProxy/State/WorkerAssignmentResult.cs @@ -8,7 +8,8 @@ namespace Azure.Functions.WorkerProxy.State; /// internal enum WorkerAssignmentResult { - Success, + Created, + AlreadyAssigned, WorkerNotReady, AssignmentConflict, WorkerTerminated diff --git a/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs index 0420ce6e3a..35b7d3db87 100644 --- a/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs +++ b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs @@ -244,7 +244,7 @@ public WorkerAssignmentResult Assign(WorkerAssignment assignment) return _state.AssignmentState == WorkerAssignmentState.Failed ? WorkerAssignmentResult.WorkerTerminated - : WorkerAssignmentResult.Success; + : WorkerAssignmentResult.AlreadyAssigned; } if (!_state.IsWorkerReady) @@ -270,7 +270,7 @@ public WorkerAssignmentResult Assign(WorkerAssignment assignment) _assignment = assignment; UpdateStateAndNotifyWaitersLocked(assigned); - return WorkerAssignmentResult.Success; + return WorkerAssignmentResult.Created; } } diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs index f113d996c4..3ada6baa97 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs @@ -21,8 +21,8 @@ namespace Azure.Functions.WorkerProxy.Tests.Management; public class ManagementApiEndpointTests { - private const string AssignPath = "/admin/worker/assign"; - private const string StatePath = "/admin/infra/instanceState"; + private const string AssignPath = "/admin/worker/assignment"; + private const string StatePath = "/admin/worker/state"; private const string ValidAssignment = """ {"functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, "environment":{"SETTING":"private-value"},"functionAppDirectory":"/home/site/wwwroot"} @@ -30,18 +30,16 @@ public class ManagementApiEndpointTests private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(30); - [Theory] - [InlineData("{}")] - [InlineData("{\"lastKnownRevision\":null}")] - public async Task InitialPollReturnsRevisionZeroImmediately(string body) + [Fact] + public async Task OmittedRevisionReturnsRevisionZeroImmediately() { await using WorkerProxyWebApplicationFactory factory = new(); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = JsonBody(body); - using HttpResponseMessage response = await client.PostAsync(StatePath, content, timeout.Token); + using HttpResponseMessage response = await client.GetAsync(StatePath, timeout.Token); Assert.Equal(HttpStatusCode.OK, response.StatusCode); + AssertNoStore(response); using JsonDocument state = JsonDocument.Parse(await response.Content.ReadAsStringAsync(timeout.Token)); Assert.Equal(0, state.RootElement.GetProperty("revisionId").GetInt64()); Assert.Equal("None", state.RootElement.GetProperty("workerPodState").GetProperty("podStatus").GetString()); @@ -49,24 +47,19 @@ public async Task InitialPollReturnsRevisionZeroImmediately(string body) } [Theory] - [InlineData(AssignPath, "")] - [InlineData(AssignPath, "null")] - [InlineData(AssignPath, "[]")] - [InlineData(AssignPath, "{")] - [InlineData(StatePath, "")] - [InlineData(StatePath, "null")] - [InlineData(StatePath, "[]")] - [InlineData(StatePath, "{")] - [InlineData(StatePath, "{\"lastKnownRevision\":\"0\"}")] - [InlineData(StatePath, "{\"lastKnownRevision\":1.5}")] - [InlineData(StatePath, "{\"lastKnownRevision\":9223372036854775808}")] - public async Task MalformedBodyUsesHostValidationEnvelope(string path, string body) + [InlineData("")] + [InlineData("null")] + [InlineData("[]")] + [InlineData("{")] + [InlineData("true")] + [InlineData("\"assignment\"")] + public async Task MalformedBodyUsesHostValidationEnvelope(string body) { await using WorkerProxyWebApplicationFactory factory = new(); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using StringContent content = JsonBody(body); - using HttpResponseMessage response = await client.PostAsync(path, content, timeout.Token); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); @@ -100,17 +93,26 @@ public async Task InvalidAssignmentDoesNotClaimIdentity(string field, string? va } await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using StringContent content = JsonBody(body.ToJsonString()); - using HttpResponseMessage response = await client.PostAsync(AssignPath, content, timeout.Token); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); await AssertValidationAsync(response, timeout.Token, (code, string.Equals(code, "InvalidBody", StringComparison.Ordinal) ? "request" : field)); - WorkerPodState state = factory.Services.GetRequiredService().State; - Assert.Equal(0, state.Revision); + WorkerPodState state = manager.State; + Assert.Equal(2, state.Revision); Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); Assert.Null(state.FunctionAppName); + + using StringContent validContent = JsonBody( + ValidAssignment.Replace("test-app", "another-app", StringComparison.Ordinal)); + using HttpResponseMessage accepted = await client.PutAsync(AssignPath, validContent, timeout.Token); + await AssertAssignmentSuccessAsync(accepted, HttpStatusCode.Created, timeout.Token); + Assert.Equal("another-app", manager.State.FunctionAppName); } [Theory] @@ -125,7 +127,7 @@ public async Task AssignmentReturnsAllInvalidFieldsInOneResponse(string body) using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using StringContent content = JsonBody(body); - using HttpResponseMessage response = await client.PostAsync(AssignPath, content, timeout.Token); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); await AssertValidationAsync(response, timeout.Token, ("Required", "functionAppName"), @@ -138,40 +140,69 @@ await AssertValidationAsync(response, timeout.Token, Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); } - [Theory] - [InlineData(AssignPath)] - [InlineData(StatePath)] - public async Task NonJsonContentUsesHostValidationEnvelope(string path) + [Fact] + public async Task NonJsonContentUsesHostValidationEnvelope() { await using WorkerProxyWebApplicationFactory factory = new(); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using StringContent content = new("{}", Encoding.UTF8, "text/plain"); - using HttpResponseMessage response = await client.PostAsync(path, content, timeout.Token); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); } [Theory] - [InlineData(-1)] - [InlineData(1)] - [InlineData(long.MaxValue)] - public async Task InvalidRevisionUsesHostValidationEnvelope(long revision) + [InlineData("lastKnownRevision=-1")] + [InlineData("lastKnownRevision=1")] + [InlineData("lastKnownRevision=9223372036854775807")] + [InlineData("lastKnownRevision=9223372036854775808")] + [InlineData("lastKnownRevision=-9223372036854775809")] + [InlineData("lastKnownRevision=-9223372036854775808")] + [InlineData("lastKnownRevision=null")] + [InlineData("lastKnownRevision=")] + [InlineData("lastKnownRevision")] + [InlineData("lastKnownRevision=0&lastKnownRevision=0")] + [InlineData("lastKnownRevision=0&lastKnownRevision=1")] + [InlineData("lastKnownRevision=0&lastKnownRevision=")] + [InlineData("lastKnownRevision=0&LASTKNOWNREVISION=0")] + [InlineData("lastKnownRevision=1.5")] + [InlineData("lastKnownRevision=0.0")] + [InlineData("lastKnownRevision=1e0")] + [InlineData("lastKnownRevision=%220%22")] + [InlineData("lastKnownRevision=true")] + [InlineData("lastKnownRevision=%200")] + [InlineData("lastKnownRevision=0%20")] + [InlineData("lastKnownRevision=%090")] + [InlineData("lastKnownRevision=0%0A")] + [InlineData("lastKnownRevision=0%00")] + [InlineData("lastKnownRevision=%000")] + [InlineData("lastKnownRevision=0%000")] + [InlineData("lastKnownRevision=%C2%A00")] + [InlineData("lastKnownRevision=%2B%200")] + [InlineData("lastKnownRevision=+0")] + [InlineData("lastKnownRevision=%2B")] + [InlineData("lastKnownRevision=-")] + [InlineData("lastKnownRevision=%2B-0")] + [InlineData("lastKnownRevision=0,0")] + [InlineData("lastKnownRevision=%D9%A0")] + public async Task InvalidRevisionUsesHostValidationEnvelope(string query) { await using WorkerProxyWebApplicationFactory factory = new(); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = JsonBody($"{{\"lastKnownRevision\":{revision}}}"); - using HttpResponseMessage response = await client.PostAsync(StatePath, content, timeout.Token); + using HttpResponseMessage response = await client.GetAsync($"{StatePath}?{query}", timeout.Token); await AssertValidationAsync(response, timeout.Token, ("InvalidRevision", "lastKnownRevision")); + AssertNoStore(response); Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); + Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); } [Theory] [InlineData("/admin/worker/ready", "GET")] - [InlineData(AssignPath, "POST")] - [InlineData(StatePath, "POST")] + [InlineData(AssignPath, "PUT")] + [InlineData(StatePath, "GET")] public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string path, string method) { await using WorkerProxyWebApplicationFactory factory = new(); @@ -183,7 +214,7 @@ public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string pa { Version = HttpVersion.Version20, VersionPolicy = HttpVersionPolicy.RequestVersionExact, - Content = string.Equals(method, "POST", StringComparison.Ordinal) ? JsonBody("{}") : null + Content = string.Equals(method, "PUT", StringComparison.Ordinal) ? JsonBody("{}") : null }; using HttpResponseMessage response = await rpcClient.SendAsync(request, timeout.Token); Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); @@ -192,7 +223,7 @@ public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string pa using HttpClient forwardingClient = factory.CreateHttpForwardingClient(); using HttpRequestMessage forwardingRequest = new(new HttpMethod(method), path) { - Content = string.Equals(method, "POST", StringComparison.Ordinal) ? JsonBody("{}") : null + Content = string.Equals(method, "PUT", StringComparison.Ordinal) ? JsonBody("{}") : null }; using HttpResponseMessage forwardingResponse = await forwardingClient.SendAsync(forwardingRequest, timeout.Token); Assert.Equal(HttpStatusCode.NotFound, forwardingResponse.StatusCode); @@ -200,8 +231,14 @@ public async Task WorkerManagementRoutesAreUnavailableOnOtherListeners(string pa [Theory] [InlineData("/admin/worker/ready", "POST")] + [InlineData("/admin/worker/ready", "PUT")] [InlineData(AssignPath, "GET")] - [InlineData(StatePath, "GET")] + [InlineData(AssignPath, "POST")] + [InlineData(AssignPath, "PATCH")] + [InlineData(AssignPath, "DELETE")] + [InlineData(StatePath, "POST")] + [InlineData(StatePath, "PUT")] + [InlineData(StatePath, "DELETE")] public async Task ManagementRoutesRejectUnsupportedMethods(string path, string method) { await using WorkerProxyWebApplicationFactory factory = new(); @@ -213,8 +250,12 @@ public async Task ManagementRoutesRejectUnsupportedMethods(string path, string m Assert.Equal(HttpStatusCode.MethodNotAllowed, response.StatusCode); } - [Fact] - public async Task EqualRevisionPollReturnsNoContentAtDeadline() + [Theory] + [InlineData("0")] + [InlineData("%2B0")] + [InlineData("-0")] + [InlineData("000")] + public async Task ExplicitZeroRevisionPollReturnsNoContentAtDeadline(string revision) { Mock clock = new(); Mock timer = new(); @@ -232,14 +273,14 @@ public async Task EqualRevisionPollReturnsNoContentAtDeadline() services.Replace(ServiceDescriptor.Singleton(clock.Object))); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = JsonBody("{\"lastKnownRevision\":0}"); - Task poll = client.PostAsync(StatePath, content, timeout.Token); + Task poll = client.GetAsync($"{StatePath}?lastKnownRevision={revision}", timeout.Token); Action fireTimer = await expire.Task.WaitAsync(timeout.Token); Assert.False(poll.IsCompleted); fireTimer(); using HttpResponseMessage response = await poll; Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + AssertNoStore(response); Assert.Empty(await response.Content.ReadAsByteArrayAsync(timeout.Token)); Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce); @@ -253,8 +294,7 @@ public async Task ClientCancellationRemovesPendingPoll() using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using CancellationTokenSource cancellation = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token); - using StringContent content = JsonBody("{\"lastKnownRevision\":0}"); - Task poll = client.PostAsync(StatePath, content, cancellation.Token); + Task poll = client.GetAsync($"{StatePath}?lastKnownRevision=0", cancellation.Token); while (manager.PendingWaiterCount == 0) { await Task.Delay(TimeSpan.FromMilliseconds(10), timeout.Token); @@ -270,6 +310,211 @@ public async Task ClientCancellationRemovesPendingPoll() Assert.Equal(0, manager.State.Revision); } + [Theory] + [InlineData("/admin/worker/assign", "POST")] + [InlineData("/admin/worker/assign", "PUT")] + [InlineData("/admin/infra/instanceState", "POST")] + [InlineData("/admin/infra/instanceState", "GET")] + public async Task ObsoleteManagementRoutesAreNotFound(string path, string method) + { + await using WorkerProxyWebApplicationFactory factory = new(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using HttpRequestMessage request = new(new HttpMethod(method), path); + using HttpResponseMessage response = await client.SendAsync(request, timeout.Token); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("0")] + [InlineData("%2B0")] + [InlineData("-0")] + [InlineData("1")] + [InlineData("%2B1")] + [InlineData("0001")] + public async Task OlderSignedInvariantRevisionReturnsCurrentSnapshot(string revision) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using HttpResponseMessage response = await client.GetAsync($"{StatePath}?lastKnownRevision={revision}", timeout.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + AssertNoStore(response); + using JsonDocument state = JsonDocument.Parse(await response.Content.ReadAsStringAsync(timeout.Token)); + Assert.Equal(2, state.RootElement.GetProperty("revisionId").GetInt64()); + Assert.Equal(2, manager.State.Revision); + Assert.Equal(0, manager.PendingWaiterCount); + } + + [Theory] + [InlineData(null, "{", "application/json")] + [InlineData(null, "not-json", "text/plain")] + [InlineData("0", "{\"lastKnownRevision\":999}", "application/json")] + [InlineData("%2B1", "not-json", "application/octet-stream")] + public async Task StateQueryIgnoresRequestBodyAndContentType(string? revision, string body, string contentType) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + string path = revision is null ? StatePath : $"{StatePath}?lastKnownRevision={revision}"; + using HttpRequestMessage request = new(HttpMethod.Get, path) + { + Content = new StringContent(body, Encoding.UTF8, contentType) + }; + using HttpResponseMessage response = await client.SendAsync(request, timeout.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + AssertNoStore(response); + using JsonDocument state = JsonDocument.Parse(await response.Content.ReadAsStringAsync(timeout.Token)); + Assert.Equal(2, state.RootElement.GetProperty("revisionId").GetInt64()); + Assert.Equal(2, manager.State.Revision); + Assert.Equal(0, manager.PendingWaiterCount); + } + + [Fact] + public async Task ConcurrentIdenticalAssignmentsCreateOnceAndReplaysDoNotChangeRevision() + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task[] assignments = Enumerable.Range(0, 12).Select(async _ => + { + await start.Task.WaitAsync(timeout.Token); + using StringContent content = JsonBody(ValidAssignment); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + Assert.True(response.StatusCode is HttpStatusCode.Created or HttpStatusCode.NoContent); + await AssertAssignmentSuccessAsync(response, response.StatusCode, timeout.Token); + return response.StatusCode; + }).ToArray(); + + start.SetResult(); + HttpStatusCode[] statuses = await Task.WhenAll(assignments).WaitAsync(timeout.Token); + Assert.Single(statuses, status => status == HttpStatusCode.Created); + Assert.Equal(statuses.Length - 1, statuses.Count(status => status == HttpStatusCode.NoContent)); + Assert.Equal(3, manager.State.Revision); + WorkerPodState assigned = manager.State; + + using StringContent replayContent = JsonBody(ValidAssignment); + using HttpResponseMessage replay = await client.PutAsync(AssignPath, replayContent, timeout.Token); + await AssertAssignmentSuccessAsync(replay, HttpStatusCode.NoContent, timeout.Token); + Assert.Same(assigned, manager.State); + } + + [Theory] + [InlineData("functionAppName", "\"TEST-app\"", false)] + [InlineData("functionGroupName", "\"TEST-group\"", false)] + [InlineData("functionAppDirectory", "\"/home/site/WWWROOT\"", false)] + [InlineData("isAlwaysReady", "true", false)] + [InlineData("environment", "{\"SETTING\":\"PRIVATE-value\"}", false)] + [InlineData("environment", "{\"setting\":\"private-value\"}", false)] + [InlineData("functionAppName", "\"TEST-app\"", true)] + [InlineData("functionGroupName", "\"TEST-group\"", true)] + [InlineData("functionAppDirectory", "\"/home/site/WWWROOT\"", true)] + [InlineData("isAlwaysReady", "true", true)] + [InlineData("environment", "{\"SETTING\":\"PRIVATE-value\"}", true)] + [InlineData("environment", "{\"setting\":\"private-value\"}", true)] + public async Task AllIdentityFieldsConflictEvenAfterTermination(string field, string value, bool terminated) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(ValidAssignment); + using HttpResponseMessage created = await client.PutAsync(AssignPath, content, timeout.Token); + await AssertAssignmentSuccessAsync(created, HttpStatusCode.Created, timeout.Token); + if (terminated) + { + Assert.True(manager.OnSessionTerminated(1)); + } + + WorkerPodState assigned = manager.State; + JsonObject different = JsonNode.Parse(ValidAssignment)!.AsObject(); + different[field] = JsonNode.Parse(value); + using StringContent conflictingContent = JsonBody(different.ToJsonString()); + using HttpResponseMessage conflict = await client.PutAsync(AssignPath, conflictingContent, timeout.Token); + await AssertAssignmentErrorAsync(conflict, HttpStatusCode.Conflict, "AssignmentConflict", timeout.Token); + Assert.Same(assigned, manager.State); + + using StringContent replayContent = JsonBody(ValidAssignment); + using HttpResponseMessage replay = await client.PutAsync(AssignPath, replayContent, timeout.Token); + if (terminated) + { + await AssertAssignmentErrorAsync(replay, HttpStatusCode.Conflict, "WorkerTerminated", timeout.Token); + } + else + { + await AssertAssignmentSuccessAsync(replay, HttpStatusCode.NoContent, timeout.Token); + } + + Assert.Same(assigned, manager.State); + } + + [Theory] + [InlineData("{}")] + [InlineData("{\"SETTING\":\"\"}")] + public async Task EmptyEnvironmentOrValueIsValidAndNotReadyDoesNotReserveIdentity(string environment) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent notReadyContent = JsonBody(ValidAssignment); + using HttpResponseMessage notReady = await client.PutAsync(AssignPath, notReadyContent, timeout.Token); + await AssertAssignmentErrorAsync(notReady, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); + Assert.Equal(0, manager.State.Revision); + Assert.Equal(WorkerAssignmentState.Unassigned, manager.State.AssignmentState); + Assert.Null(manager.State.FunctionAppName); + + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + JsonObject assignment = JsonNode.Parse(ValidAssignment)!.AsObject(); + assignment["functionAppName"] = "other-app"; + assignment["environment"] = JsonNode.Parse(environment); + using StringContent content = JsonBody(assignment.ToJsonString()); + using HttpResponseMessage created = await client.PutAsync(AssignPath, content, timeout.Token); + await AssertAssignmentSuccessAsync(created, HttpStatusCode.Created, timeout.Token); + Assert.Equal(3, manager.State.Revision); + Assert.Equal("other-app", manager.State.FunctionAppName); + } + + private static void AssertNoStore(HttpResponseMessage response) => + Assert.True(response.Headers.CacheControl?.NoStore); + + private static async Task AssertAssignmentSuccessAsync( + HttpResponseMessage response, HttpStatusCode statusCode, CancellationToken cancellationToken) + { + Assert.Equal(statusCode, response.StatusCode); + Assert.Empty(await response.Content.ReadAsByteArrayAsync(cancellationToken)); + if (statusCode == HttpStatusCode.Created) + { + Assert.Equal(AssignPath, response.Headers.Location?.OriginalString); + } + } + + private static async Task AssertAssignmentErrorAsync( + HttpResponseMessage response, HttpStatusCode statusCode, string code, CancellationToken cancellationToken) + { + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); + using JsonDocument json = JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); + Assert.Equal("error", Assert.Single(json.RootElement.EnumerateObject()).Name); + Assert.Equal(code, json.RootElement.GetProperty("error").GetProperty("code").GetString()); + } + private static StringContent JsonBody(string body) => new(body, Encoding.UTF8, "application/json"); private static async Task AssertValidationAsync( diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs index 92833fb7b4..0a753f8ad9 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs @@ -36,7 +36,7 @@ public void GetWorkerReady_TracksStartStreamAndTerminationWithoutChangingState() Assert.Same(started, manager.State); Assert.Equal(WorkerPodStatus.None, started.PodStatus); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); Assert.IsType(ManagementApiHandlers.GetWorkerReady(manager)); manager.OnSessionTerminated(1); WorkerPodState terminated = manager.State; @@ -103,7 +103,7 @@ public void AssignWorker_ReturnsAllBlankFieldsAndInvalidEnvironmentTogether() ], errors); Assert.Same(before, manager.State); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); } [Theory] @@ -155,7 +155,7 @@ public void AssignWorker_InvalidFieldsDoNotClaimAssignment(string field, string? Assert.Equal(field is "environmentKey" or "environmentValue" ? "InvalidValue" : "Required", detail.Code); Assert.Same(before, manager.State); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); Assert.Equal("other-app", manager.State.FunctionAppName); } @@ -170,7 +170,9 @@ public void AssignWorker_ExplicitBooleanAndEmptyEnvironmentAreAccepted(bool isAl IResult result = ManagementApiHandlers.AssignWorker( CreateRequest(isAlwaysReady: isAlwaysReady, environment: new()), manager); - Assert.Equal(200, Assert.IsType(result).StatusCode); + Created created = Assert.IsType(result); + Assert.Equal(201, created.StatusCode); + Assert.Equal("/admin/worker/assignment", created.Location); Assert.Equal(before.Revision + 1, manager.State.Revision); Assert.Equal(WorkerAssignmentState.Ready, manager.State.AssignmentState); Assert.Equal(WorkerPodStatus.ReadyForRequest, manager.State.PodStatus); @@ -188,7 +190,7 @@ public void AssignWorker_EmptyValuesAndNonemptyWhitespaceKeysAreAccepted() { WorkerPodStateManager manager = CreateReadyManager(); - Assert.IsType(ManagementApiHandlers.AssignWorker( + Assert.IsType(ManagementApiHandlers.AssignWorker( CreateRequest(environment: new() { ["SETTING"] = string.Empty, [" "] = string.Empty }), manager)); } @@ -199,7 +201,7 @@ public void AssignWorker_RecordsEnvironmentWithoutApplyingItToProxyProcess() string setting = $"WORKERPROXY_ASSIGNMENT_TEST_{Guid.NewGuid():N}"; Assert.Null(Environment.GetEnvironmentVariable(setting)); - Assert.IsType(ManagementApiHandlers.AssignWorker( + Assert.IsType(ManagementApiHandlers.AssignWorker( CreateRequest(environment: new() { [setting] = "private-value" }), manager)); Assert.Null(Environment.GetEnvironmentVariable(setting)); @@ -227,7 +229,7 @@ public void AssignWorker_NotReadyDoesNotReserveIdentity(bool attached) } manager.OnWorkerStartStream(1, "worker"); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other-app"), manager)); Assert.Equal("other-app", manager.State.FunctionAppName); } @@ -237,7 +239,7 @@ public void AssignWorker_CopiesEnvironmentAndReplaysEquivalentIdentityRegardless WorkerPodStateManager manager = CreateReadyManager(); Dictionary environment = new() { ["SETTING"] = "private-value", ["EMPTY"] = string.Empty }; WorkerAssignRequest request = CreateRequest(environment: environment); - Assert.IsType(ManagementApiHandlers.AssignWorker(request, manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(request, manager)); WorkerPodState assigned = manager.State; environment["SETTING"] = "changed-value"; @@ -248,7 +250,7 @@ public void AssignWorker_CopiesEnvironmentAndReplaysEquivalentIdentityRegardless ["SETTING"] = "private-value" }); - Assert.IsType(ManagementApiHandlers.AssignWorker(equivalent, manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(equivalent, manager)); AssertError(ManagementApiHandlers.AssignWorker(request, manager), 409, "AssignmentConflict"); Assert.Same(assigned, manager.State); } @@ -263,7 +265,7 @@ public void AssignWorker_CopiesEnvironmentAndReplaysEquivalentIdentityRegardless public void AssignWorker_DifferentIdentityConflictsBeforeAndAfterTermination(string field) { WorkerPodStateManager manager = CreateReadyManager(); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); WorkerAssignRequest different = field switch { "app" => CreateRequest(functionAppName: "APP"), @@ -282,25 +284,10 @@ public void AssignWorker_DifferentIdentityConflictsBeforeAndAfterTermination(str WorkerPodState terminated = manager.State; AssertError(ManagementApiHandlers.AssignWorker(different, manager), 409, "AssignmentConflict"); - AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(), manager), 503, "WorkerTerminated"); + AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(), manager), 409, "WorkerTerminated"); Assert.Same(terminated, manager.State); } - [Fact] - public async Task GetInstanceStateAsync_NullRequestReturnsInvalidBodyWithoutWaiting() - { - PollClock clock = new(); - WorkerPodStateManager manager = CreateManager(clock.Provider); - WorkerPodState initial = manager.State; - - Assert.Equal(new("InvalidBody", "request"), Assert.Single( - AssertValidation(await ManagementApiHandlers.GetInstanceStateAsync(null, manager)))); - - Assert.Same(initial, manager.State); - Assert.Equal(0, manager.PendingWaiterCount); - Assert.Empty(clock.Timers); - } - [Theory] [InlineData(false)] [InlineData(true)] @@ -312,11 +299,11 @@ public async Task GetInstanceStateAsync_UnspecifiedRevisionReturnsImmediateImmut { manager.OnWorkerAttached(1); manager.OnWorkerStartStream(1, "worker"); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); } WorkerPodState before = manager.State; - Task poll = ManagementApiHandlers.GetInstanceStateAsync(new(), manager); + Task poll = ManagementApiHandlers.GetInstanceStateAsync(null, manager); Assert.True(poll.IsCompletedSuccessfully); WorkerInstanceState response = AssertState(await poll, before); @@ -344,7 +331,7 @@ public async Task GetInstanceStateAsync_StaleRevisionReturnsCurrentSnapshotImmed WorkerPodStateManager manager = CreateManager(clock.Provider); manager.OnWorkerAttached(1); - Task poll = ManagementApiHandlers.GetInstanceStateAsync(new() { LastKnownRevision = 0 }, manager); + Task poll = ManagementApiHandlers.GetInstanceStateAsync(0, manager); Assert.True(poll.IsCompletedSuccessfully); AssertState(await poll, manager.State); @@ -364,7 +351,7 @@ public async Task GetInstanceStateAsync_InvalidRevisionReturnsErrorWithoutWaitin Assert.Equal(new("InvalidRevision", "lastKnownRevision"), Assert.Single( AssertValidation(await ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = revision }, manager)))); + revision, manager)))); Assert.Same(initial, manager.State); Assert.Equal(0, manager.PendingWaiterCount); @@ -392,11 +379,11 @@ public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapsho if (string.Equals(transition, "terminate", StringComparison.Ordinal)) { - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); } long revision = manager.State.Revision; - Task poll = ManagementApiHandlers.GetInstanceStateAsync(new() { LastKnownRevision = revision }, manager); + Task poll = ManagementApiHandlers.GetInstanceStateAsync(revision, manager); Assert.False(poll.IsCompleted); Assert.Equal(1, manager.PendingWaiterCount); @@ -409,7 +396,7 @@ public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapsho manager.OnWorkerStartStream(1, "worker"); break; case "assign": - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); break; case "terminate": manager.OnSessionTerminated(1); @@ -429,12 +416,12 @@ public async Task GetInstanceStateAsync_ReplaysAndConflictsDoNotWakePollBeforeSi { PollClock clock = new(); WorkerPodStateManager manager = CreateReadyManager(clock.Provider); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); WorkerPodState assigned = manager.State; Task poll = ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = assigned.Revision }, manager); + assigned.Revision, manager); - Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(functionAppName: "other"), manager), 409, "AssignmentConflict"); Assert.False(poll.IsCompleted); @@ -458,9 +445,9 @@ public async Task GetInstanceStateAsync_CancellationPropagatesAndLeavesOtherPoll WorkerPodStateManager manager = CreateManager(clock.Provider); using CancellationTokenSource cancellation = new(); Task canceledPoll = ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = 0 }, manager, cancellation.Token); + 0, manager, cancellation.Token); Task activePoll = ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = 0 }, manager); + 0, manager); Assert.Equal(2, manager.PendingWaiterCount); cancellation.Cancel(); @@ -484,7 +471,7 @@ public async Task GetInstanceStateAsync_CancelingOnlyPollRemovesWaiterWithoutCha WorkerPodState initial = manager.State; using CancellationTokenSource cancellation = new(); Task poll = ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = 0 }, manager, cancellation.Token); + 0, manager, cancellation.Token); Assert.Equal(1, manager.PendingWaiterCount); cancellation.Cancel(); @@ -509,7 +496,7 @@ public async Task GetInstanceStateAsync_PreCanceledRequestPropagatesWithoutRegis OperationCanceledException exception = await Assert.ThrowsAnyAsync( () => ManagementApiHandlers.GetInstanceStateAsync( - new() { LastKnownRevision = revision }, manager, cancellation.Token)); + revision, manager, cancellation.Token)); Assert.Equal(cancellation.Token, exception.CancellationToken); Assert.Equal(0, manager.PendingWaiterCount); diff --git a/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs index 35e6e25696..1e23047855 100644 --- a/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs @@ -103,53 +103,10 @@ public void Assignment_MalformedTypesThrowJsonException(string json) JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); } - [Theory] - [InlineData("{}", null)] - [InlineData("""{"lastKnownRevision":null}""", null)] - [InlineData("""{"lastKnownRevision":0}""", 0L)] - [InlineData("""{"LASTKNOWNREVISION":17}""", 17L)] - [InlineData("""{"lastKnownRevision":-1}""", -1L)] - [InlineData("""{"lastKnownRevision":9223372036854775807}""", long.MaxValue)] - public void PollRequest_PreservesNullableAndExactIntegerRevision(string json, long? expected) - { - InstanceStatePollRequest request = Assert.IsType( - JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.InstanceStatePollRequest)); - - Assert.Equal(expected, request.LastKnownRevision); - string serialized = JsonSerializer.Serialize(request, WorkerProxyJsonContext.Default.InstanceStatePollRequest); - using JsonDocument document = JsonDocument.Parse(serialized); - if (expected.HasValue) - { - AssertProperties(document.RootElement, "lastKnownRevision"); - Assert.Equal(JsonValueKind.Number, document.RootElement.GetProperty("lastKnownRevision").ValueKind); - Assert.Equal(expected.Value, document.RootElement.GetProperty("lastKnownRevision").GetInt64()); - } - else - { - AssertProperties(document.RootElement); - } - } - - [Theory] - [InlineData("""{"lastKnownRevision":"0"}""")] - [InlineData("""{"lastKnownRevision":true}""")] - [InlineData("""{"lastKnownRevision":1.5}""")] - [InlineData("""{"lastKnownRevision":9223372036854775808}""")] - [InlineData("""{"lastKnownRevision":-9223372036854775809}""")] - [InlineData("""{"lastKnownRevision":{}}""")] - [InlineData("[]")] - [InlineData("{")] - public void PollRequest_MalformedOrOutOfRangeRevisionThrowsJsonException(string json) - { - Assert.Throws(() => - JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.InstanceStatePollRequest)); - } - [Fact] - public void NullRequests_DeserializeAsNullForHandlerValidation() + public void NullAssignment_DeserializesAsNullForHandlerValidation() { Assert.Null(JsonSerializer.Deserialize("null", WorkerProxyJsonContext.Default.WorkerAssignRequest)); - Assert.Null(JsonSerializer.Deserialize("null", WorkerProxyJsonContext.Default.InstanceStatePollRequest)); } [Fact] diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs index 040929a19c..0d51a747c9 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.CapabilityFinalization.cs @@ -88,7 +88,7 @@ public async Task Relay_FinalizerFailureTerminatesAssignedWorkerWithoutForwardin await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); await ExchangeAsync(runtime, worker, "attach", timeout.Token); WorkerAssignment assignment = CreateWorkerAssignment(); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(assignment)); Task poll = manager.WaitForChangeAsync(manager.State.Revision, timeout.Token); await worker.WriteAsync(CreateInitResponse("init", null), timeout.Token); @@ -129,7 +129,7 @@ public async Task Relay_ShutdownDuringFinalizationDoesNotRestoreDestinationOrRea await using RelayClient runtime = CreateClient(factory, FunctionRpcRelaySide.Runtime, timeout.Token); await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); await ExchangeAsync(runtime, worker, "attach", timeout.Token); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateWorkerAssignment())); try { diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs index 7e80aa6f2d..454ccfc92b 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Globalization; using System.Net; using System.Net.Http; using System.Text; @@ -36,8 +37,8 @@ public async Task ManagementApis_ObserveNormalStartupAssignmentAndTerminalFailur using CancellationTokenSource timeout = new(TestTimeout); await AssertReadinessAsync(management, "/admin/instance/ready", HttpStatusCode.OK, timeout.Token); await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); - using (HttpResponseMessage notReady = await PostManagementJsonAsync( - management, "/admin/worker/assign", ManagementAssignment, timeout.Token)) + using (HttpResponseMessage notReady = await PutManagementJsonAsync( + management, "/admin/worker/assignment", ManagementAssignment, timeout.Token)) { await AssertManagementErrorAsync(notReady, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); } @@ -69,25 +70,27 @@ await WaitForAttachmentAsync(factory.Services.GetRequiredService assignmentPoll = PostManagementJsonAsync( - management, "/admin/infra/instanceState", "{\"lastKnownRevision\":2}", timeout.Token); + Task assignmentPoll = management.GetAsync( + "/admin/worker/state?lastKnownRevision=2", timeout.Token); await WaitForManagementPollAsync(manager, timeout.Token); - using (HttpResponseMessage assignment = await PostManagementJsonAsync( - management, "/admin/worker/assign", ManagementAssignment, timeout.Token)) + using (HttpResponseMessage assignment = await PutManagementJsonAsync( + management, "/admin/worker/assignment", ManagementAssignment, timeout.Token)) { - Assert.Equal(HttpStatusCode.OK, assignment.StatusCode); + Assert.Equal(HttpStatusCode.Created, assignment.StatusCode); + Assert.Equal("/admin/worker/assignment", assignment.Headers.Location?.OriginalString); Assert.Empty(await assignment.Content.ReadAsByteArrayAsync(timeout.Token)); } using (HttpResponseMessage changed = await assignmentPoll) { Assert.Equal(HttpStatusCode.OK, changed.StatusCode); + Assert.True(changed.Headers.CacheControl?.NoStore); string body = await changed.Content.ReadAsStringAsync(timeout.Token); using JsonDocument state = JsonDocument.Parse(body); Assert.Equal("FunctionsWorkerPod", state.RootElement.GetProperty("functionsContainerType").GetString()); @@ -104,16 +107,16 @@ await WaitForAttachmentAsync(factory.Services.GetRequiredService terminationPoll = PostManagementJsonAsync( - management, "/admin/infra/instanceState", "{\"lastKnownRevision\":3}", timeout.Token); + Task terminationPoll = management.GetAsync( + "/admin/worker/state?lastKnownRevision=3", timeout.Token); await WaitForManagementPollAsync(manager, timeout.Token); await worker.CompleteRequestAsync(timeout.Token); Assert.Equal(StatusCode.Unavailable, await runtime.WaitForTerminationAsync(timeout.Token)); @@ -143,6 +148,7 @@ await WaitForAttachmentAsync(factory.Services.GetRequiredService PostManagementJsonAsync( + private static async Task PutManagementJsonAsync( HttpClient client, string path, string body, CancellationToken cancellationToken) { using StringContent content = new(body, Encoding.UTF8, "application/json"); - return await client.PostAsync(path, content, cancellationToken); + return await client.PutAsync(path, content, cancellationToken); } private static async Task ReadManagementStateAsync( - HttpClient client, string body, CancellationToken cancellationToken) + HttpClient client, long? lastKnownRevision, CancellationToken cancellationToken) { - using HttpResponseMessage response = await PostManagementJsonAsync( - client, "/admin/infra/instanceState", body, cancellationToken); + string path = lastKnownRevision.HasValue + ? "/admin/worker/state?lastKnownRevision=" + lastKnownRevision.Value.ToString(CultureInfo.InvariantCulture) + : "/admin/worker/state"; + using HttpResponseMessage response = await client.GetAsync(path, cancellationToken); Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.True(response.Headers.CacheControl?.NoStore); return JsonDocument.Parse(await response.Content.ReadAsStringAsync(cancellationToken)); } @@ -207,6 +219,10 @@ private static async Task AssertReadinessAsync( using HttpResponseMessage response = await client.GetAsync(path, cancellationToken); Assert.Equal(statusCode, response.StatusCode); Assert.Empty(await response.Content.ReadAsByteArrayAsync(cancellationToken)); + if (string.Equals(path, "/admin/worker/ready", StringComparison.Ordinal)) + { + Assert.True(response.Headers.CacheControl?.NoStore); + } } private static async Task AssertManagementErrorAsync( diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs index 410e823d2a..169df7638f 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Readiness.cs @@ -60,7 +60,7 @@ public async Task Relay_ReadinessRequiresStartStreamButNotRuntimeInitialization( Assert.Equal(start.StartStream.WorkerId, ready.WorkerId); Assert.Equal(WorkerPodStatus.None, ready.PodStatus); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateWorkerAssignment())); Assert.Equal(WorkerPodStatus.ReadyForRequest, manager.State.PodStatus); Assert.Equal(3, manager.State.Revision); Assert.False(workerOutbound.Reader.TryRead(out _)); @@ -167,7 +167,7 @@ public async Task Relay_TerminationWithdrawsReadinessBeforeBlockedWriterReleases await workerInbound.Writer.WriteAsync(CreateStartStream(), timeout.Token); await blockingWriter.WriteEntered.WaitAsync(timeout.Token); WorkerAssignment assignment = CreateWorkerAssignment(); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(assignment)); WorkerPodState assigned = manager.State; Task poll = manager.WaitForChangeAsync(assigned.Revision, timeout.Token); @@ -233,7 +233,7 @@ public async Task Relay_ReplacementReadinessRespectsTerminalAssignment(bool assi if (assigned) { - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(assignment)); } await worker.CompleteRequestAsync(timeout.Token); @@ -256,7 +256,7 @@ public async Task Relay_ReplacementReadinessRespectsTerminalAssignment(bool assi Assert.True(manager.State.IsWorkerReady); Assert.True(manager.State.SessionId > terminated.SessionId); Assert.Equal(terminated.Revision + 2, manager.State.Revision); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(assignment)); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(assignment)); } } diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs index 62b6f92be6..f49a6c3b04 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs @@ -202,7 +202,7 @@ public async Task Relay_ReconnectDuringSessionTeardownReturnsUnavailable() await using RelayClient worker = CreateClient(factory, FunctionRpcRelaySide.Worker, timeout.Token); await ExchangeAsync(runtime, worker, "teardown", timeout.Token); WorkerPodStateManager manager = factory.Services.GetRequiredService(); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateWorkerAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateWorkerAssignment())); Task poll = manager.WaitForChangeAsync(manager.State.Revision, timeout.Token); await runtime.CompleteRequestAsync(timeout.Token); diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs index e6b0175faa..80acaceff9 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs @@ -212,7 +212,7 @@ public async Task Poll_NoOpNotificationsAndAssignmentRetriesDoNotWakeWaiter() Assert.False(manager.OnWorkerAttached(1)); Assert.False(manager.OnWorkerStartStream(1, "worker")); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.AlreadyAssigned, manager.Assign(CreateAssignment())); Assert.Equal(WorkerAssignmentResult.AssignmentConflict, manager.Assign(CreateAssignment("other"))); Assert.False(poll.IsCompleted); Assert.Equal(1, manager.PendingWaiterCount); diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs index 8055cec280..2da8597919 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs @@ -46,7 +46,7 @@ public void Lifecycle_PublishesImmutableMonotonicSnapshots() Assert.True(started.IsWorkerReady); Assert.Equal(WorkerPodStatus.None, started.PodStatus); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment())); WorkerPodState assigned = manager.State; Assert.Equal(WorkerAssignmentState.Ready, assigned.AssignmentState); Assert.Equal(WorkerPodStatus.ReadyForRequest, assigned.PodStatus); @@ -86,7 +86,7 @@ public void NotReadyAssignment_DoesNotClaimIdentityOrChangeRevision() Assert.Same(attached, manager.State); manager.OnWorkerStartStream(1, "worker"); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment("accepted"))); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment("accepted"))); Assert.Equal("accepted", manager.State.FunctionAppName); } @@ -94,10 +94,10 @@ public void NotReadyAssignment_DoesNotClaimIdentityOrChangeRevision() public void AssignmentReplayAndConflict_DoNotChangeRevision() { WorkerPodStateManager manager = CreateReadyManager(); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment())); WorkerPodState state = manager.State; - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.AlreadyAssigned, manager.Assign(CreateAssignment())); Assert.Equal(WorkerAssignmentResult.AssignmentConflict, manager.Assign(CreateAssignment("other"))); Assert.Same(state, manager.State); } @@ -137,7 +137,7 @@ public void UnassignedSession_CanBeReplacedWithoutResettingRevision() Assert.False(manager.OnSessionTerminated(1)); Assert.Same(replacement, manager.State); Assert.Equal(5, replacement.Revision); - Assert.Equal(WorkerAssignmentResult.Success, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment())); Assert.Equal(6, manager.State.Revision); } @@ -258,9 +258,10 @@ public async Task ConcurrentEquivalentAssignments_PublishOnlyOnce() })).ToArray(); start.SetResult(); - WorkerAssignmentResult[] results = await Task.WhenAll(attempts); + WorkerAssignmentResult[] results = await Task.WhenAll(attempts).WaitAsync(TestTimeout); - Assert.All(results, result => Assert.Equal(WorkerAssignmentResult.Success, result)); + Assert.Single(results, result => result == WorkerAssignmentResult.Created); + Assert.Equal(31, results.Count(result => result == WorkerAssignmentResult.AlreadyAssigned)); Assert.Equal(3, manager.State.Revision); } @@ -277,8 +278,8 @@ public async Task ConcurrentConflictingAssignments_HaveOneWinningIdentity() })).ToArray(); start.SetResult(); - (string AppName, WorkerAssignmentResult Result)[] results = await Task.WhenAll(attempts); - (string AppName, WorkerAssignmentResult Result) winner = Assert.Single(results, result => result.Result == WorkerAssignmentResult.Success); + (string AppName, WorkerAssignmentResult Result)[] results = await Task.WhenAll(attempts).WaitAsync(TestTimeout); + (string AppName, WorkerAssignmentResult Result) winner = Assert.Single(results, result => result.Result == WorkerAssignmentResult.Created); Assert.Equal(winner.AppName, manager.State.FunctionAppName); Assert.Equal(31, results.Count(result => result.Result == WorkerAssignmentResult.AssignmentConflict)); @@ -309,7 +310,7 @@ public async Task AssignmentRacingTermination_CannotLeaveReadyState() WorkerAssignmentResult result = await assignment; Assert.False(manager.State.IsWorkerReady); Assert.Equal(WorkerPodStatus.None, manager.State.PodStatus); - if (result == WorkerAssignmentResult.Success) + if (result == WorkerAssignmentResult.Created) { Assert.Equal(WorkerAssignmentState.Failed, manager.State.AssignmentState); Assert.Equal(WorkerAssignmentResult.WorkerTerminated, manager.Assign(CreateAssignment())); From 1141b1709829a92375c3c18bdcb422932ce8e0f1 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 16:26:24 -0700 Subject: [PATCH 06/10] Fix assignment charset validation and synchronize timer disposal assertions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Management/ManagementApiEndpoints.cs | 17 +++++- .../Management/ManagementApiEndpointTests.cs | 57 +++++++++++++++++++ .../Management/ManagementApiHandlersTests.cs | 18 ++++-- .../WorkerPodStateManagerTests.Polling.cs | 17 ++++-- .../WorkerProxyApplicationTests.cs | 3 + 5 files changed, 100 insertions(+), 12 deletions(-) diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs index 78834ee783..cdf7ce3704 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -3,6 +3,7 @@ using System; using System.Globalization; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.State; @@ -17,7 +18,7 @@ namespace Azure.Functions.WorkerProxy.Management; /// /// /// The PUT handler explicitly uses ReadFromJsonAsync rather than automatic body binding so malformed JSON, -/// incompatible field types, and unsupported content types return our Host-aligned HTTP 400 InvalidBody +/// incompatible field types, and unsupported content types or charsets return our Host-aligned HTTP 400 InvalidBody /// validation envelope. Automatic binding can reject requests before the handler runs with framework-owned /// 400/415 responses that do not guarantee that envelope. /// @@ -45,6 +46,20 @@ private static async Task AssignWorkerAsync(HttpRequest request, Worker return ManagementApiHandlers.InvalidBody(); } + var charset = request.GetTypedHeaders().ContentType!.Charset; + if (charset.HasValue && !charset.Equals("utf-8", StringComparison.OrdinalIgnoreCase)) + { + // ReadFromJsonAsync wraps unsupported encodings in InvalidOperationException, not JsonException. + try + { + _ = Encoding.GetEncoding(charset.Value); + } + catch (Exception exception) when (exception is ArgumentException or NotSupportedException) + { + return ManagementApiHandlers.InvalidBody(); + } + } + WorkerAssignRequest? assignment; try { diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs index 3ada6baa97..4aefd82e5f 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs @@ -152,6 +152,60 @@ public async Task NonJsonContentUsesHostValidationEnvelope() await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); } + [Theory] + [InlineData("application/json; charset=not-a-real-charset", "{}")] + [InlineData("application/json; charset=not-a-real-charset", ValidAssignment)] + [InlineData("application/json; charset=\"not-a-real-charset\"", ValidAssignment)] + [InlineData("application/problem+json; charset=not-a-real-charset", ValidAssignment)] + [InlineData("application/json; charset=utf-7", ValidAssignment)] + public async Task UnsupportedJsonCharsetUsesHostValidationEnvelope(string contentType, string body) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + WorkerPodState before = manager.State; + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(body); + content.Headers.Remove("Content-Type"); + Assert.True(content.Headers.TryAddWithoutValidation("Content-Type", contentType)); + + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + + await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); + Assert.Same(before, manager.State); + using StringContent validContent = JsonBody(ValidAssignment); + using HttpResponseMessage accepted = await client.PutAsync(AssignPath, validContent, timeout.Token); + await AssertAssignmentSuccessAsync(accepted, HttpStatusCode.Created, timeout.Token); + } + + [Theory] + [InlineData("application/json", "utf-8")] + [InlineData("application/json; charset=utf-8", "utf-8")] + [InlineData("application/json; charset=UTF-8", "utf-8")] + [InlineData("application/json; charset=utf-16", "utf-16")] + [InlineData("application/problem+json; charset=utf-16", "utf-16")] + public async Task SupportedJsonCharsetPreservesAssignment(string contentType, string encodingName) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + const string appName = "test-app-\u00e9"; + string body = ValidAssignment.Replace("test-app", appName, StringComparison.Ordinal); + using StringContent content = new(body, Encoding.GetEncoding(encodingName)); + content.Headers.Remove("Content-Type"); + Assert.True(content.Headers.TryAddWithoutValidation("Content-Type", contentType)); + + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + + await AssertAssignmentSuccessAsync(response, HttpStatusCode.Created, timeout.Token); + Assert.Equal(appName, manager.State.FunctionAppName); + } + [Theory] [InlineData("lastKnownRevision=-1")] [InlineData("lastKnownRevision=1")] @@ -259,6 +313,8 @@ public async Task ExplicitZeroRevisionPollReturnsNoContentAtDeadline(string revi { Mock clock = new(); Mock timer = new(); + TaskCompletionSource disposed = new(TaskCreationOptions.RunContinuationsAsynchronously); + timer.Setup(instance => instance.Dispose()).Callback(() => disposed.TrySetResult()); TaskCompletionSource expire = new(TaskCreationOptions.RunContinuationsAsynchronously); clock.Setup(provider => provider.CreateTimer( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) @@ -283,6 +339,7 @@ public async Task ExplicitZeroRevisionPollReturnsNoContentAtDeadline(string revi AssertNoStore(response); Assert.Empty(await response.Content.ReadAsByteArrayAsync(timeout.Token)); Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); + await disposed.Task.WaitAsync(timeout.Token); timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce); } diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs index 0a753f8ad9..f2ed360146 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.Management; @@ -408,7 +409,7 @@ public async Task GetInstanceStateAsync_EqualRevisionWaitsForTypedChangedSnapsho WorkerInstanceState response = AssertState(await poll.WaitAsync(TestTimeout), manager.State); Assert.Equal(revision + 1, response.RevisionId); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -435,7 +436,7 @@ public async Task GetInstanceStateAsync_ReplaysAndConflictsDoNotWakePollBeforeSi Assert.Equal(204, Assert.IsType(await poll.WaitAsync(TestTimeout)).StatusCode); Assert.Same(assigned, manager.State); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -460,7 +461,7 @@ public async Task GetInstanceStateAsync_CancellationPropagatesAndLeavesOtherPoll manager.OnWorkerAttached(1); AssertState(await activePoll.WaitAsync(TestTimeout), manager.State); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -481,7 +482,7 @@ public async Task GetInstanceStateAsync_CancelingOnlyPollRemovesWaiterWithoutCha Assert.Equal(cancellation.Token, exception.CancellationToken); Assert.Same(initial, manager.State); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Theory] @@ -569,6 +570,7 @@ public PollClock() .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => { ScheduledTimer timer = new(callback, state, dueTime, period); + timer.Timer.Setup(instance => instance.Dispose()).Callback(() => timer.Disposed.TrySetResult()); Timers.Add(timer); return timer.Timer.Object; }); @@ -579,13 +581,19 @@ public PollClock() public List Timers { get; } = []; - public void VerifyTimersDisposed() => + public async Task VerifyTimersDisposedAsync() + { + // WaitAsync may complete its promise before disposing its timer. + await Task.WhenAll(Timers.Select(timer => timer.Disposed.Task)).WaitAsync(TestTimeout); Assert.All(Timers, timer => timer.Timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce())); + } public sealed record ScheduledTimer(TimerCallback Callback, object? State, TimeSpan DueTime, TimeSpan Period) { public Mock Timer { get; } = new(); + public TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Fire() => Callback(State); } } diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs index 80acaceff9..c7b076bdaf 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.Polling.cs @@ -108,7 +108,7 @@ public async Task Poll_EachPublishedTransitionNotifiesAllWaiters(string transiti Assert.Equal(revision + 1, result.State!.Revision); }); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -133,7 +133,7 @@ public async Task Poll_UnchangedStateReturnsNoChangeAtSixtySecondDeadline() Assert.Null(result.State); Assert.Same(initial, manager.State); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -158,7 +158,7 @@ public async Task Poll_CancellationRemovesOnlyCanceledWaiter() manager.OnWorkerAttached(1); Assert.True((await activePoll.WaitAsync(TestTimeout)).HasChanged); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Fact] @@ -177,7 +177,7 @@ public async Task Poll_TimeoutRemovesOnlyExpiredWaiter() manager.OnWorkerAttached(1); Assert.True((await activePoll.WaitAsync(TestTimeout)).HasChanged); Assert.Equal(0, manager.PendingWaiterCount); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } [Theory] @@ -321,7 +321,7 @@ public async Task Poll_TimeoutRacingPublicationAlwaysCleansUp() Assert.Equal(0, manager.PendingWaiterCount); Assert.Same(manager.State, (await manager.WaitForChangeAsync(0, timeout.Token)).State); - clock.VerifyTimersDisposed(); + await clock.VerifyTimersDisposedAsync(); } } @@ -350,6 +350,7 @@ public PollClock() .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => { ScheduledTimer timer = new(callback, state, dueTime, period); + timer.Timer.Setup(instance => instance.Dispose()).Callback(() => timer.Disposed.TrySetResult()); Timers.Add(timer); return timer.Timer.Object; }); @@ -360,8 +361,10 @@ public PollClock() public List Timers { get; } = []; - public void VerifyTimersDisposed() + public async Task VerifyTimersDisposedAsync() { + // WaitAsync may complete its promise before disposing its timer. + await Task.WhenAll(Timers.Select(timer => timer.Disposed.Task)).WaitAsync(TestTimeout); Assert.All(Timers, timer => timer.Timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce())); } @@ -369,6 +372,8 @@ public sealed record ScheduledTimer(TimerCallback Callback, object? State, TimeS { public Mock Timer { get; } = new(); + public TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Fire() => Callback(State); } } diff --git a/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs b/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs index 6de71a0cba..5f4c58bd25 100644 --- a/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs +++ b/test/Functions.WorkerProxy.Tests/WorkerProxyApplicationTests.cs @@ -50,6 +50,8 @@ public async Task StateManager_UsesTimeProviderFromContainer() { Mock provider = new(); Mock timer = new(); + TaskCompletionSource disposed = new(TaskCreationOptions.RunContinuationsAsynchronously); + timer.Setup(instance => instance.Dispose()).Callback(() => disposed.TrySetResult()); provider.Setup(clock => clock.CreateTimer( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(timer.Object); @@ -65,6 +67,7 @@ public async Task StateManager_UsesTimeProviderFromContainer() It.IsAny(), It.IsAny(), TimeSpan.FromSeconds(60), Timeout.InfiniteTimeSpan), Times.Once()); manager.OnWorkerAttached(1); Assert.True((await poll).HasChanged); + await disposed.Task.WaitAsync(timeout.Token); timer.Verify(instance => instance.Dispose(), Times.AtLeastOnce()); } From 67a85fc6d97c45af53e3f9303ab2ef2a75a1f4ed Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 11 Sep 2026 19:52:11 -0700 Subject: [PATCH 07/10] Make relay shutdown test logging gates deterministic Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../FunctionRpcRelayTests.HttpCapabilities.cs | 2 +- .../Rpc/FunctionRpcRelayTests.TestDoubles.cs | 11 +++--- .../Rpc/FunctionRpcRelayTests.cs | 37 +++++++++++++++++-- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs index 4aa93f7a4b..57060ec661 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.HttpCapabilities.cs @@ -285,7 +285,7 @@ public async Task Relay_ReplacementSession_DoesNotReusePreviousDestination(strin [Fact] public async Task Relay_TerminalSession_MakesHttpUnavailableBeforeAttachmentsRelease() { - using BlockingLogger logger = new(); + using BlockingLogger logger = new(eventIdToBlock: SessionTerminatedEventId); Dictionary configuration = new() { [$"{WorkerProxyOptions.SectionName}:{nameof(WorkerProxyOptions.HttpProxyEndpoint)}"] = "http://worker-pod:28080/" diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.TestDoubles.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.TestDoubles.cs index f646dd2066..dc88526229 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.TestDoubles.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.TestDoubles.cs @@ -109,14 +109,14 @@ public void Release() } } - private sealed class BlockingLogger : ILogger, IDisposable + private sealed class BlockingLogger(int? eventIdToBlock = null) : ILogger, IDisposable { private readonly ManualResetEventSlim _release = new(initialState: false); - private readonly TaskCompletionSource _logEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _logEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _logCount; - public Task LogEntered => _logEntered.Task; + public Task LogEntered => _logEntered.Task; public IDisposable? BeginScope(TState state) where TState : notnull @@ -131,9 +131,10 @@ public bool IsEnabled(LogLevel logLevel) public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { - if (Interlocked.Increment(ref _logCount) == 1) + if ((eventIdToBlock is null || eventId.Id == eventIdToBlock) + && Interlocked.Increment(ref _logCount) == 1) { - _logEntered.TrySetResult(true); + _logEntered.TrySetResult(eventId); _release.Wait(); } } diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs index f49a6c3b04..4f4872255a 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.cs @@ -17,6 +17,7 @@ using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Xunit; @@ -28,6 +29,7 @@ public partial class FunctionRpcRelayTests { private const string FunctionRpcServiceName = "AzureFunctionsRpcMessages.FunctionRpc"; private const string EventStreamMethodName = "EventStream"; + private const int SessionTerminatedEventId = 1; private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(30); private static readonly Marshaller StreamingMessageMarshaller = @@ -193,7 +195,7 @@ public async Task Relay_PeerCloseStopsSiblingAndAllowsWholeSessionReplacement() [Fact] public async Task Relay_ReconnectDuringSessionTeardownReturnsUnavailable() { - using BlockingLogger logger = new(); + using BlockingLogger logger = new(eventIdToBlock: SessionTerminatedEventId); await using WorkerProxyWebApplicationFactory factory = new( configureServices: services => services.AddSingleton>(logger)); FunctionRpcRelay relay = factory.Services.GetRequiredService(); @@ -281,7 +283,7 @@ public async Task Relay_FirstReadFaultStopsBothStreamOperationsAndPreservesFault [Fact] public async Task Relay_CanceledStopWaitDoesNotCancelSharedStop() { - using BlockingLogger logger = new(); + using BlockingLogger logger = new(eventIdToBlock: SessionTerminatedEventId); FunctionRpcRelay relay = new(logger, CreateCapabilityProvider(), CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); using CancellationTokenSource stopCancellation = new(); @@ -391,7 +393,7 @@ public async Task Relay_ConcurrentStopsJoinSharedCompletion() [Fact] public async Task Relay_ShutdownAllowsSessionClearBeforeCancellation() { - using BlockingLogger logger = new(); + using BlockingLogger logger = new(eventIdToBlock: SessionTerminatedEventId); FunctionRpcRelay relay = new(logger, CreateCapabilityProvider(), CreatePodStateManager()); using CancellationTokenSource timeout = new(TestTimeout); Task runtimeTask = @@ -408,6 +410,10 @@ public async Task Relay_ShutdownAllowsSessionClearBeforeCancellation() FunctionRpcRelayTerminalState[] terminalStates = await Task.WhenAll(runtimeTask, workerTask).WaitAsync(timeout.Token); Assert.All(terminalStates, static state => Assert.Equal(FunctionRpcRelayTerminationReason.Shutdown, state.Reason)); + Assert.False(relay.IsAttached(FunctionRpcRelaySide.Runtime)); + Assert.False(relay.IsAttached(FunctionRpcRelaySide.Worker)); + Assert.Equal(FunctionRpcRelayTerminationReason.Shutdown, relay.LastTerminalState?.Reason); + Assert.False(stopTask.IsCompleted); } finally { @@ -419,6 +425,31 @@ public async Task Relay_ShutdownAllowsSessionClearBeforeCancellation() await relay.DisposeAsync(); } + [Fact] + public async Task BlockingLogger_SecondaryFailureDoesNotConsumeTerminationGate() + { + using BlockingLogger logger = new(eventIdToBlock: SessionTerminatedEventId); + using CancellationTokenSource timeout = new(TestTimeout); + Task logging = Task.Run(() => + { + // Completing the channels can let a secondary log race ahead of the termination log. + logger.Log(LogLevel.Debug, new EventId(2), "Secondary stream failure", null, static (state, _) => state); + logger.Log(LogLevel.Debug, new EventId(SessionTerminatedEventId), "Session terminated", null, static (state, _) => state); + }, timeout.Token); + + try + { + EventId blockedEvent = await logger.LogEntered.WaitAsync(timeout.Token); + Assert.Equal(SessionTerminatedEventId, blockedEvent.Id); + Assert.False(logging.IsCompleted); + } + finally + { + logger.Release(); + await logging.WaitAsync(TestTimeout); + } + } + private static WorkerProxyWebApplicationFactory CreateFactory() { return new WorkerProxyWebApplicationFactory(); From fa48ddb1fc46e7d159eb49ed195509510f5901ec Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Sat, 12 Sep 2026 08:41:18 -0700 Subject: [PATCH 08/10] Supply pod identity to WorkerProxy CI smoke container Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/ci/templates/jobs/run-linux-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/ci/templates/jobs/run-linux-tests.yml b/eng/ci/templates/jobs/run-linux-tests.yml index b1bf0657d5..d867aa8487 100644 --- a/eng/ci/templates/jobs/run-linux-tests.yml +++ b/eng/ci/templates/jobs/run-linux-tests.yml @@ -57,6 +57,7 @@ jobs: echo "Starting WorkerProxy image: $(localImage)" containerId="$(docker run --detach \ --publish 127.0.0.1::80 \ + --env WORKERPROXY__PODNAME=ci-worker-pod \ $(localImage))" echo "Started WorkerProxy container: $containerId" cleanup() { From 103d21ad5e57b354fca5e07a07ed578aac8b18eb Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Mon, 14 Sep 2026 21:53:30 -0700 Subject: [PATCH 09/10] Align WorkerProxy startup mode validation and API handlers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Management/ManagementApiEndpoints.cs | 78 +----- .../Management/ManagementApiHandlers.cs | 100 +++----- .../Management/WorkerApiErrorCodes.cs | 1 - .../Management/WorkerAssignRequest.cs | 17 +- .../WorkerAssignRequestValidator.cs | 119 +++++++++ .../Management/WorkerInstanceState.cs | 2 +- .../Management/WorkerPodStateResponse.cs | 1 + .../State/WorkerAssignment.cs | 26 +- .../State/WorkerPodState.cs | 2 + .../State/WorkerPodStateManager.cs | 2 + .../State/WorkerStartupMode.cs | 20 ++ .../Management/ManagementApiEndpointTests.cs | 238 +++++++++++++++--- .../Management/ManagementApiHandlersTests.cs | 149 +++++++++-- .../Management/WorkerProxyJsonContextTests.cs | 56 ++++- .../Rpc/FunctionRpcRelayTests.Management.cs | 24 +- .../Rpc/FunctionRpcRelayTests.Readiness.cs | 2 +- .../State/WorkerAssignmentTests.cs | 52 +++- .../State/WorkerPodStateManagerTests.cs | 45 +++- 18 files changed, 716 insertions(+), 218 deletions(-) create mode 100644 src/Functions.WorkerProxy/Management/WorkerAssignRequestValidator.cs create mode 100644 src/Functions.WorkerProxy/State/WorkerStartupMode.cs diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs index cdf7ce3704..992a97bf10 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -1,10 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; -using System.Globalization; -using System.Text; -using System.Text.Json; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.State; using Microsoft.AspNetCore.Builder; @@ -17,80 +13,26 @@ namespace Azure.Functions.WorkerProxy.Management; /// Registers worker lifecycle APIs on the management listener. /// /// -/// The PUT handler explicitly uses ReadFromJsonAsync rather than automatic body binding so malformed JSON, -/// incompatible field types, and unsupported content types or charsets return our Host-aligned HTTP 400 InvalidBody -/// validation envelope. Automatic binding can reject requests before the handler runs with framework-owned -/// 400/415 responses that do not guarantee that envelope. +/// Platform callers are expected to send UTF-8 JSON. Assignment uses framework JSON binding; +/// binding failures follow framework behavior without a guaranteed status or error envelope. +/// Successfully bound requests use our field-validation envelope and lifecycle error codes. /// internal static class ManagementApiEndpoints { public static void Map(IEndpointRouteBuilder endpoints) { endpoints.MapGet("/admin/worker/ready", ManagementApiHandlers.GetWorkerReady) - .AddEndpointFilter(DisableCaching).AllowAnonymous(); - endpoints.MapPut("/admin/worker/assignment", AssignWorkerAsync).AllowAnonymous(); - endpoints.MapGet("/admin/worker/state", GetInstanceStateAsync) - .AddEndpointFilter(DisableCaching).AllowAnonymous(); + .AddEndpointFilter(DisableResponseCaching).AllowAnonymous(); + endpoints.MapPut("/admin/worker/assignment", + (WorkerAssignRequest request, WorkerPodStateManager manager) => ManagementApiHandlers.AssignWorker(request, manager)) + .AllowAnonymous(); + endpoints.MapGet("/admin/worker/state", ManagementApiHandlers.GetWorkerStateAsync) + .AddEndpointFilter(DisableResponseCaching).AllowAnonymous(); } - private static ValueTask DisableCaching(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + private static ValueTask DisableResponseCaching(EndpointFilterInvocationContext context, EndpointFilterDelegate next) { context.HttpContext.Response.Headers.CacheControl = "no-store"; return next(context); } - - private static async Task AssignWorkerAsync(HttpRequest request, WorkerPodStateManager manager) - { - if (!request.HasJsonContentType()) - { - return ManagementApiHandlers.InvalidBody(); - } - - var charset = request.GetTypedHeaders().ContentType!.Charset; - if (charset.HasValue && !charset.Equals("utf-8", StringComparison.OrdinalIgnoreCase)) - { - // ReadFromJsonAsync wraps unsupported encodings in InvalidOperationException, not JsonException. - try - { - _ = Encoding.GetEncoding(charset.Value); - } - catch (Exception exception) when (exception is ArgumentException or NotSupportedException) - { - return ManagementApiHandlers.InvalidBody(); - } - } - - WorkerAssignRequest? assignment; - try - { - assignment = await request.ReadFromJsonAsync( - WorkerProxyJsonContext.Default.WorkerAssignRequest, request.HttpContext.RequestAborted); - } - catch (JsonException) - { - return ManagementApiHandlers.InvalidBody(); - } - - return ManagementApiHandlers.AssignWorker(assignment, manager); - } - - private static async Task GetInstanceStateAsync(HttpRequest request, WorkerPodStateManager manager) - { - long? lastKnownRevision = null; - if (request.Query.TryGetValue("lastKnownRevision", out var revisions)) - { - string? value = revisions.Count == 1 ? revisions[0] : null; - // Accept invariant ASCII digits with an optional leading sign, not whitespace or JSON syntax. - if (value is null - || value.AsSpan().IndexOfAnyExcept("+-0123456789") >= 0 - || !long.TryParse(value, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out long revision)) - { - return ManagementApiHandlers.InvalidRevision(); - } - - lastKnownRevision = revision; - } - - return await ManagementApiHandlers.GetInstanceStateAsync(lastKnownRevision, manager, request.HttpContext.RequestAborted); - } } diff --git a/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs index fb9a256cbe..63c580870f 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Azure.Functions.WorkerProxy.State; @@ -18,69 +19,14 @@ internal static class ManagementApiHandlers public static IResult GetWorkerReady(WorkerPodStateManager manager) => manager.State.IsWorkerReady ? TypedResults.Ok() : TypedResults.StatusCode(StatusCodes.Status503ServiceUnavailable); - public static IResult AssignWorker(WorkerAssignRequest? request, WorkerPodStateManager manager) + public static IResult AssignWorker(WorkerAssignRequest request, WorkerPodStateManager manager) { - if (request is null) - { - return InvalidBody(); - } - - List errors = []; - if (string.IsNullOrWhiteSpace(request.FunctionAppName)) - { - errors.Add(new(WorkerApiErrorCodes.Required, "functionAppName")); - } - - if (string.IsNullOrWhiteSpace(request.FunctionGroupName)) - { - errors.Add(new(WorkerApiErrorCodes.Required, "functionGroupName")); - } - - if (request.IsAlwaysReady is null) - { - errors.Add(new(WorkerApiErrorCodes.Required, "isAlwaysReady")); - } - - if (string.IsNullOrWhiteSpace(request.FunctionAppDirectory)) - { - errors.Add(new(WorkerApiErrorCodes.Required, "functionAppDirectory")); - } - - Dictionary environment = new(StringComparer.Ordinal); - if (request.Environment is null) - { - errors.Add(new(WorkerApiErrorCodes.Required, "environment")); - } - else - { - foreach ((string key, string? value) in request.Environment) - { - if (string.IsNullOrEmpty(key) || value is null) - { - // Report the invalid field once without exposing environment keys or values. - errors.Add(new(WorkerApiErrorCodes.InvalidValue, "environment")); - break; - } - - environment.Add(key, value); - } - } - - if (errors.Count > 0 - || request.FunctionAppName is not { } functionAppName - || request.FunctionGroupName is not { } functionGroupName - || request.FunctionAppDirectory is not { } functionAppDirectory - || request.IsAlwaysReady is not { } isAlwaysReady) + if (!WorkerAssignRequestValidator.TryCreateAssignment( + request, out WorkerAssignment? assignment, out IReadOnlyList errors)) { return ValidationError(errors); } - WorkerAssignment assignment = new( - functionAppName, - functionGroupName, - isAlwaysReady, - environment, - functionAppDirectory); return manager.Assign(assignment) switch { WorkerAssignmentResult.Created => TypedResults.Created("/admin/worker/assignment"), @@ -95,6 +41,19 @@ public static IResult AssignWorker(WorkerAssignRequest? request, WorkerPodStateM }; } + /// + /// Parses the optional revision query and returns or polls worker state, honoring request cancellation. + /// + public static async Task GetWorkerStateAsync(HttpRequest request, WorkerPodStateManager manager) + { + if (!TryGetLastKnownRevision(request.Query, out long? lastKnownRevision)) + { + return InvalidRevision(); + } + + return await GetInstanceStateAsync(lastKnownRevision, manager, request.HttpContext.RequestAborted); + } + public static async Task GetInstanceStateAsync( long? revision, WorkerPodStateManager manager, @@ -116,12 +75,31 @@ public static async Task GetInstanceStateAsync( return result.State is { } state ? StateResponse(state) : TypedResults.NoContent(); } - internal static IResult InvalidBody() => - ValidationError([new(WorkerApiErrorCodes.InvalidBody, "request")]); - internal static IResult InvalidRevision() => ValidationError([new(WorkerApiErrorCodes.InvalidRevision, "lastKnownRevision")]); + private static bool TryGetLastKnownRevision(IQueryCollection query, out long? lastKnownRevision) + { + lastKnownRevision = null; + if (!query.TryGetValue("lastKnownRevision", out var revisions)) + { + return true; + } + + if (revisions is not [string value]) + { + return false; + } + + if (!long.TryParse(value, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out long revision)) + { + return false; + } + + lastKnownRevision = revision; + return true; + } + private static IResult ValidationError(IReadOnlyList errors) => TypedResults.Json(new RequestValidationResponse(errors), WorkerProxyJsonContext.Default.RequestValidationResponse, statusCode: StatusCodes.Status400BadRequest); diff --git a/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs b/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs index 414c266413..d27f1c908c 100644 --- a/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs +++ b/src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs @@ -11,7 +11,6 @@ internal static class WorkerApiErrorCodes // Clients branch on these exact, case-sensitive wire values. Do not change or repurpose them. // Keep explicit literals rather than nameof so symbol renames cannot change the contract. public const string Required = "Required"; - public const string InvalidBody = "InvalidBody"; public const string InvalidValue = "InvalidValue"; public const string InvalidRevision = "InvalidRevision"; public const string WorkerNotReady = "WorkerNotReady"; diff --git a/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs b/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs index af40cea706..125e32386b 100644 --- a/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs +++ b/src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs @@ -6,10 +6,25 @@ namespace Azure.Functions.WorkerProxy.Management; /// -/// Describes assignment identity for an already-specialized worker. +/// Describes assignment identity and the caller-selected startup mode. /// +/// +/// All fields are required. The directory may be empty for Preconfigured but must be nonblank +/// for SpecializationRequired. Configuration is currently retained for retry comparison only. +/// internal sealed class WorkerAssignRequest { + /// + /// Gets the required, case-sensitive startup mode selected by the caller. + /// + /// + /// Preconfigured means the worker starts with its final application configuration (BYOC). + /// SpecializationRequired means the worker requires configuration through specialization. + /// Assignment currently records either mode without performing specialization. + /// The raw string is retained so invalid values can be reported as field-level validation errors. + /// + public string? StartupMode { get; init; } + public string? FunctionAppName { get; init; } public string? FunctionGroupName { get; init; } diff --git a/src/Functions.WorkerProxy/Management/WorkerAssignRequestValidator.cs b/src/Functions.WorkerProxy/Management/WorkerAssignRequestValidator.cs new file mode 100644 index 0000000000..cdf8bb0815 --- /dev/null +++ b/src/Functions.WorkerProxy/Management/WorkerAssignRequestValidator.cs @@ -0,0 +1,119 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Azure.Functions.WorkerProxy.State; + +namespace Azure.Functions.WorkerProxy.Management; + +/// +/// Validates bound assignment requests and captures immutable identity without accessing worker state. +/// +internal static class WorkerAssignRequestValidator +{ + /// + /// Creates an assignment from a valid request, or collects ordered field errors without exposing environment entries. + /// + public static bool TryCreateAssignment( + WorkerAssignRequest request, + [NotNullWhen(true)] out WorkerAssignment? assignment, + out IReadOnlyList validationErrors) + { + assignment = null; + List errors = []; + validationErrors = errors; + WorkerStartupMode? startupMode = ValidateStartupMode(request.StartupMode, errors); + + if (string.IsNullOrWhiteSpace(request.FunctionAppName)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionAppName")); + } + + if (string.IsNullOrWhiteSpace(request.FunctionGroupName)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionGroupName")); + } + + if (request.IsAlwaysReady is null) + { + errors.Add(new(WorkerApiErrorCodes.Required, "isAlwaysReady")); + } + + if (request.FunctionAppDirectory is null + || (startupMode is WorkerStartupMode.SpecializationRequired && string.IsNullOrWhiteSpace(request.FunctionAppDirectory))) + { + errors.Add(new(WorkerApiErrorCodes.Required, "functionAppDirectory")); + } + + Dictionary environment = ValidateAndCopyEnvironment(request.Environment, errors); + + if (errors.Count > 0 + || startupMode is not { } selectedStartupMode + || request.FunctionAppName is not { } functionAppName + || request.FunctionGroupName is not { } functionGroupName + || request.FunctionAppDirectory is not { } functionAppDirectory + || request.IsAlwaysReady is not { } isAlwaysReady) + { + return false; + } + + assignment = new( + selectedStartupMode, + functionAppName, + functionGroupName, + isAlwaysReady, + environment, + functionAppDirectory); + + return true; + } + + private static WorkerStartupMode? ValidateStartupMode(string? value, List errors) + { + if (string.IsNullOrWhiteSpace(value)) + { + errors.Add(new(WorkerApiErrorCodes.Required, "startupMode")); + return null; + } + + if (string.Equals(value, nameof(WorkerStartupMode.Preconfigured), StringComparison.Ordinal)) + { + return WorkerStartupMode.Preconfigured; + } + + if (string.Equals(value, nameof(WorkerStartupMode.SpecializationRequired), StringComparison.Ordinal)) + { + return WorkerStartupMode.SpecializationRequired; + } + + errors.Add(new(WorkerApiErrorCodes.InvalidValue, "startupMode")); + return null; + } + + private static Dictionary ValidateAndCopyEnvironment( + IReadOnlyDictionary? source, List errors) + { + Dictionary environment = new(StringComparer.Ordinal); + if (source is null) + { + errors.Add(new(WorkerApiErrorCodes.Required, "environment")); + return environment; + } + + foreach ((string key, string? value) in source) + { + if (string.IsNullOrEmpty(key) || value is null) + { + // Report the invalid field once without exposing environment keys or values. + errors.Add(new(WorkerApiErrorCodes.InvalidValue, "environment")); + break; + } + + environment.Add(key, value); + } + + return environment; + } +} diff --git a/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs b/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs index feace916af..fecf3cfbdc 100644 --- a/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs +++ b/src/Functions.WorkerProxy/Management/WorkerInstanceState.cs @@ -16,5 +16,5 @@ internal sealed record WorkerInstanceState( public string FunctionsContainerType => "FunctionsWorkerPod"; public static WorkerInstanceState FromState(WorkerPodState state) => - new(state.PodName, state.Revision, new(state.PodStatus, state.FunctionGroupName, state.IsAlwaysReady)); + new(state.PodName, state.Revision, new(state.PodStatus, state.StartupMode, state.FunctionGroupName, state.IsAlwaysReady)); } diff --git a/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs b/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs index c486ead7ee..548dba534f 100644 --- a/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs +++ b/src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs @@ -11,5 +11,6 @@ namespace Azure.Functions.WorkerProxy.Management; /// internal sealed record WorkerPodStateResponse( [property: JsonConverter(typeof(JsonStringEnumConverter))] WorkerPodStatus PodStatus, + [property: JsonConverter(typeof(JsonStringEnumConverter))] WorkerStartupMode? StartupMode, string? FunctionGroupName, bool? IsAlwaysReady); diff --git a/src/Functions.WorkerProxy/State/WorkerAssignment.cs b/src/Functions.WorkerProxy/State/WorkerAssignment.cs index f4710466d2..5a9c043e63 100644 --- a/src/Functions.WorkerProxy/State/WorkerAssignment.cs +++ b/src/Functions.WorkerProxy/State/WorkerAssignment.cs @@ -8,7 +8,7 @@ namespace Azure.Functions.WorkerProxy.State; /// -/// Captures immutable assignment identity for an already-specialized worker. +/// Captures immutable assignment identity, including the caller-selected startup mode. /// /// /// Environment and app directory participate in retry equality only; this model does not apply them @@ -20,19 +20,30 @@ internal sealed class WorkerAssignment /// Initializes a new instance of the class. /// /// - /// Copies environment entries using ordinal key equality. + /// Copies environment entries using ordinal key equality. A directory is required in both modes, + /// but only SpecializationRequired requires it to be nonblank. No filesystem check is performed. /// public WorkerAssignment( + WorkerStartupMode startupMode, string functionAppName, string functionGroupName, bool isAlwaysReady, IReadOnlyDictionary environment, string functionAppDirectory) { + if (startupMode is not (WorkerStartupMode.Preconfigured or WorkerStartupMode.SpecializationRequired)) + { + throw new ArgumentOutOfRangeException(nameof(startupMode)); + } + ArgumentException.ThrowIfNullOrWhiteSpace(functionAppName); ArgumentException.ThrowIfNullOrWhiteSpace(functionGroupName); ArgumentNullException.ThrowIfNull(environment); - ArgumentException.ThrowIfNullOrWhiteSpace(functionAppDirectory); + ArgumentNullException.ThrowIfNull(functionAppDirectory); + if (startupMode is WorkerStartupMode.SpecializationRequired) + { + ArgumentException.ThrowIfNullOrWhiteSpace(functionAppDirectory); + } foreach ((string key, string value) in environment) { @@ -40,6 +51,7 @@ public WorkerAssignment( ArgumentNullException.ThrowIfNull(value); } + StartupMode = startupMode; FunctionAppName = functionAppName; FunctionGroupName = functionGroupName; IsAlwaysReady = isAlwaysReady; @@ -48,6 +60,11 @@ public WorkerAssignment( FunctionAppDirectory = functionAppDirectory; } + /// + /// Gets the startup mode fixed by the accepted assignment. + /// + public WorkerStartupMode StartupMode { get; } + public string FunctionAppName { get; } public string FunctionGroupName { get; } @@ -65,7 +82,8 @@ public bool IsEquivalentTo(WorkerAssignment other) { ArgumentNullException.ThrowIfNull(other); - if (!string.Equals(FunctionAppName, other.FunctionAppName, StringComparison.Ordinal) + if (StartupMode != other.StartupMode + || !string.Equals(FunctionAppName, other.FunctionAppName, StringComparison.Ordinal) || !string.Equals(FunctionGroupName, other.FunctionGroupName, StringComparison.Ordinal) || IsAlwaysReady != other.IsAlwaysReady || !string.Equals(FunctionAppDirectory, other.FunctionAppDirectory, StringComparison.Ordinal) diff --git a/src/Functions.WorkerProxy/State/WorkerPodState.cs b/src/Functions.WorkerProxy/State/WorkerPodState.cs index 9f23b17954..dc082cccc2 100644 --- a/src/Functions.WorkerProxy/State/WorkerPodState.cs +++ b/src/Functions.WorkerProxy/State/WorkerPodState.cs @@ -13,6 +13,7 @@ internal sealed record WorkerPodState( bool IsWorkerAttached, string? WorkerId, WorkerAssignmentState AssignmentState, + WorkerStartupMode? StartupMode, string? FunctionAppName, string? FunctionGroupName, bool? IsAlwaysReady) @@ -36,6 +37,7 @@ public static WorkerPodState CreateUnassigned(string podName) => IsWorkerAttached: false, WorkerId: null, AssignmentState: WorkerAssignmentState.Unassigned, + StartupMode: null, FunctionAppName: null, FunctionGroupName: null, IsAlwaysReady: null); diff --git a/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs index 35b7d3db87..e60c342c21 100644 --- a/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs +++ b/src/Functions.WorkerProxy/State/WorkerPodStateManager.cs @@ -15,6 +15,7 @@ namespace Azure.Functions.WorkerProxy.State; /// /// Lifecycle notifications use the relay's monotonically increasing session IDs. /// Assignment is synchronous bookkeeping; it never initializes or specializes a worker. +/// Both startup modes are retained as identity; mode-specific preparation is not implemented. /// All state decisions and snapshot replacements share one lock, including assignment racing with termination. /// Readers can retain a snapshot after releasing the lock because subsequent transitions replace rather than mutate it. /// @@ -257,6 +258,7 @@ public WorkerAssignmentResult Assign(WorkerAssignment assignment) { Revision = checked(_state.Revision + 1), AssignmentState = WorkerAssignmentState.Ready, + StartupMode = assignment.StartupMode, FunctionAppName = assignment.FunctionAppName, FunctionGroupName = assignment.FunctionGroupName, IsAlwaysReady = assignment.IsAlwaysReady diff --git a/src/Functions.WorkerProxy/State/WorkerStartupMode.cs b/src/Functions.WorkerProxy/State/WorkerStartupMode.cs new file mode 100644 index 0000000000..bb27524eb2 --- /dev/null +++ b/src/Functions.WorkerProxy/State/WorkerStartupMode.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Azure.Functions.WorkerProxy.State; + +/// +/// Identifies the immutable startup mode selected by an assignment. +/// +internal enum WorkerStartupMode +{ + /// + /// The worker starts with its final configuration. + /// + Preconfigured, + + /// + /// The worker requires configuration through specialization. + /// + SpecializationRequired +} diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs index 4aefd82e5f..4326849ac1 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiEndpointTests.cs @@ -10,6 +10,7 @@ using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; +using Azure.Functions.WorkerProxy.Management; using Azure.Functions.WorkerProxy.Rpc; using Azure.Functions.WorkerProxy.State; using Microsoft.Extensions.DependencyInjection; @@ -24,7 +25,7 @@ public class ManagementApiEndpointTests private const string AssignPath = "/admin/worker/assignment"; private const string StatePath = "/admin/worker/state"; private const string ValidAssignment = """ - {"functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, + {"startupMode":"SpecializationRequired","functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, "environment":{"SETTING":"private-value"},"functionAppDirectory":"/home/site/wwwroot"} """; @@ -43,46 +44,94 @@ public async Task OmittedRevisionReturnsRevisionZeroImmediately() using JsonDocument state = JsonDocument.Parse(await response.Content.ReadAsStringAsync(timeout.Token)); Assert.Equal(0, state.RootElement.GetProperty("revisionId").GetInt64()); Assert.Equal("None", state.RootElement.GetProperty("workerPodState").GetProperty("podStatus").GetString()); + Assert.False(state.RootElement.GetProperty("workerPodState").TryGetProperty("startupMode", out _)); Assert.Equal(0, factory.Services.GetRequiredService().PendingWaiterCount); } [Theory] + [InlineData(null)] [InlineData("")] [InlineData("null")] [InlineData("[]")] [InlineData("{")] [InlineData("true")] [InlineData("\"assignment\"")] - public async Task MalformedBodyUsesHostValidationEnvelope(string body) + public async Task InvalidBodyReturnsBadRequestWithoutChangingAssignment(string? body) { await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + WorkerPodState before = manager.State; using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = JsonBody(body); + using StringContent? content = body is null ? null : JsonBody(body); using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); - await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); - Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + Assert.Same(before, manager.State); + Assert.Null(manager.State.StartupMode); + + using StringContent validContent = JsonBody(ValidAssignment); + using HttpResponseMessage accepted = await client.PutAsync(AssignPath, validContent, timeout.Token); + await AssertAssignmentSuccessAsync(accepted, HttpStatusCode.Created, timeout.Token); + WorkerPodState assigned = manager.State; + + using StringContent? rejectedContent = body is null ? null : JsonBody(body); + using HttpResponseMessage rejected = await client.PutAsync(AssignPath, rejectedContent, timeout.Token); + Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode); + Assert.Same(assigned, manager.State); + + using StringContent replayContent = JsonBody(ValidAssignment); + using HttpResponseMessage replay = await client.PutAsync(AssignPath, replayContent, timeout.Token); + await AssertAssignmentSuccessAsync(replay, HttpStatusCode.NoContent, timeout.Token); + Assert.Same(assigned, manager.State); } [Theory] + [InlineData("startupMode", null, "Required")] + [InlineData("startupMode", "null", "Required")] + [InlineData("startupMode", "\"\"", "Required")] + [InlineData("startupMode", "\" \\t\\r\\n\"", "Required")] + [InlineData("startupMode", "\"preconfigured\"", "InvalidValue")] + [InlineData("startupMode", "\"specializationRequired\"", "InvalidValue")] + [InlineData("startupMode", "\"Preconfigured \"", "InvalidValue")] + [InlineData("startupMode", "\" SpecializationRequired\"", "InvalidValue")] + [InlineData("startupMode", "\"Unknown\"", "InvalidValue")] + [InlineData("startupMode", "\"0\"", "InvalidValue")] + [InlineData("startupMode", "0", null)] + [InlineData("startupMode", "true", null)] + [InlineData("startupMode", "{}", null)] + [InlineData("startupMode", "[]", null)] [InlineData("functionAppName", null, "Required")] [InlineData("functionAppName", "\" \"", "Required")] [InlineData("functionGroupName", null, "Required")] [InlineData("functionGroupName", "\"\"", "Required")] [InlineData("functionAppDirectory", null, "Required")] + [InlineData("functionAppDirectory", "null", "Required")] + [InlineData("functionAppDirectory", "\"\"", "Required")] [InlineData("functionAppDirectory", "\" \"", "Required")] + [InlineData("functionAppDirectory", null, "Required", "Preconfigured")] + [InlineData("functionAppDirectory", "null", "Required", "Preconfigured")] [InlineData("isAlwaysReady", null, "Required")] [InlineData("isAlwaysReady", "null", "Required")] - [InlineData("isAlwaysReady", "\"false\"", "InvalidBody")] + [InlineData("isAlwaysReady", "\"false\"", null)] [InlineData("environment", null, "Required")] [InlineData("environment", "null", "Required")] [InlineData("environment", "{\"\":\"private-value\"}", "InvalidValue")] [InlineData("environment", "{\"SETTING\":null}", "InvalidValue")] - [InlineData("environment", "{\"SETTING\":123}", "InvalidBody")] - public async Task InvalidAssignmentDoesNotClaimIdentity(string field, string? value, string code) + [InlineData("environment", "{\"SETTING\":123}", null)] + [InlineData("environment", null, "Required", "Preconfigured")] + [InlineData("environment", "null", "Required", "Preconfigured")] + [InlineData("environment", "[]", null, "Preconfigured")] + [InlineData("environment", "{\"\":\"private-value\"}", "InvalidValue", "Preconfigured")] + [InlineData("environment", "{\"SETTING\":null}", "InvalidValue", "Preconfigured")] + [InlineData("environment", "{\"SETTING\":123}", null, "Preconfigured")] + public async Task InvalidAssignmentDoesNotClaimIdentity( + string field, string? value, string? code, string startupMode = "SpecializationRequired") { JsonObject body = JsonNode.Parse(ValidAssignment)!.AsObject(); + body["startupMode"] = startupMode; if (value is null) { body.Remove(field); @@ -96,29 +145,52 @@ public async Task InvalidAssignmentDoesNotClaimIdentity(string field, string? va WorkerPodStateManager manager = factory.Services.GetRequiredService(); Assert.True(manager.OnWorkerAttached(1)); Assert.True(manager.OnWorkerStartStream(1, "worker")); + WorkerPodState before = manager.State; using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); using StringContent content = JsonBody(body.ToJsonString()); using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); - await AssertValidationAsync(response, timeout.Token, - (code, string.Equals(code, "InvalidBody", StringComparison.Ordinal) ? "request" : field)); + if (code is null) + { + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + else + { + await AssertValidationAsync(response, timeout.Token, (code, field)); + } WorkerPodState state = manager.State; + Assert.Same(before, state); Assert.Equal(2, state.Revision); Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); Assert.Null(state.FunctionAppName); + Assert.Null(state.StartupMode); using StringContent validContent = JsonBody( ValidAssignment.Replace("test-app", "another-app", StringComparison.Ordinal)); using HttpResponseMessage accepted = await client.PutAsync(AssignPath, validContent, timeout.Token); await AssertAssignmentSuccessAsync(accepted, HttpStatusCode.Created, timeout.Token); Assert.Equal("another-app", manager.State.FunctionAppName); + + WorkerPodState assigned = manager.State; + using StringContent rejectedContent = JsonBody(body.ToJsonString()); + using HttpResponseMessage rejected = await client.PutAsync(AssignPath, rejectedContent, timeout.Token); + if (code is null) + { + Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode); + } + else + { + await AssertValidationAsync(rejected, timeout.Token, (code, field)); + } + + Assert.Same(assigned, manager.State); } [Theory] - [InlineData("{}")] + [InlineData("""{"startupMode":"SpecializationRequired"}""")] [InlineData(""" - {"functionAppName":" ","functionGroupName":"","isAlwaysReady":null, + {"startupMode":"SpecializationRequired","functionAppName":" ","functionGroupName":"","isAlwaysReady":null, "functionAppDirectory":" ","environment":{"PRIVATE_SETTING":null}} """)] public async Task AssignmentReturnsAllInvalidFieldsInOneResponse(string body) @@ -129,36 +201,27 @@ public async Task AssignmentReturnsAllInvalidFieldsInOneResponse(string body) using StringContent content = JsonBody(body); using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + string environmentError = string.Equals(body, """{"startupMode":"SpecializationRequired"}""", StringComparison.Ordinal) + ? "Required" : "InvalidValue"; await AssertValidationAsync(response, timeout.Token, ("Required", "functionAppName"), ("Required", "functionGroupName"), ("Required", "isAlwaysReady"), ("Required", "functionAppDirectory"), - (string.Equals(body, "{}", StringComparison.Ordinal) ? "Required" : "InvalidValue", "environment")); + (environmentError, "environment")); string json = await response.Content.ReadAsStringAsync(timeout.Token); Assert.DoesNotContain("PRIVATE_SETTING", json); Assert.Equal(0, factory.Services.GetRequiredService().State.Revision); } - [Fact] - public async Task NonJsonContentUsesHostValidationEnvelope() - { - await using WorkerProxyWebApplicationFactory factory = new(); - using HttpClient client = factory.CreateWorkerProxyClient(); - using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = new("{}", Encoding.UTF8, "text/plain"); - using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); - - await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); - } - [Theory] - [InlineData("application/json; charset=not-a-real-charset", "{}")] - [InlineData("application/json; charset=not-a-real-charset", ValidAssignment)] - [InlineData("application/json; charset=\"not-a-real-charset\"", ValidAssignment)] - [InlineData("application/problem+json; charset=not-a-real-charset", ValidAssignment)] - [InlineData("application/json; charset=utf-7", ValidAssignment)] - public async Task UnsupportedJsonCharsetUsesHostValidationEnvelope(string contentType, string body) + [InlineData(null)] + [InlineData("text/plain")] + [InlineData("application/xml")] + [InlineData("application/octet-stream")] + [InlineData("text/json")] + [InlineData("text/plain; charset=not-a-real-charset")] + public async Task UnsupportedMediaTypeReturns415WithoutChangingAssignment(string? contentType) { await using WorkerProxyWebApplicationFactory factory = new(); WorkerPodStateManager manager = factory.Services.GetRequiredService(); @@ -167,21 +230,76 @@ public async Task UnsupportedJsonCharsetUsesHostValidationEnvelope(string conten WorkerPodState before = manager.State; using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); - using StringContent content = JsonBody(body); + using StringContent content = JsonBody(ValidAssignment); content.Headers.Remove("Content-Type"); - Assert.True(content.Headers.TryAddWithoutValidation("Content-Type", contentType)); + if (contentType is not null) + { + Assert.True(content.Headers.TryAddWithoutValidation("Content-Type", contentType)); + } using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); - await AssertValidationAsync(response, timeout.Token, ("InvalidBody", "request")); + Assert.Equal(HttpStatusCode.UnsupportedMediaType, response.StatusCode); Assert.Same(before, manager.State); + Assert.Null(manager.State.StartupMode); + using StringContent validContent = JsonBody(ValidAssignment); using HttpResponseMessage accepted = await client.PutAsync(AssignPath, validContent, timeout.Token); await AssertAssignmentSuccessAsync(accepted, HttpStatusCode.Created, timeout.Token); + WorkerPodState assigned = manager.State; + + JsonObject different = JsonNode.Parse(ValidAssignment)!.AsObject(); + different["startupMode"] = "Preconfigured"; + different["functionAppName"] = "other-app"; + using StringContent conflictingContent = JsonBody(different.ToJsonString()); + conflictingContent.Headers.Remove("Content-Type"); + if (contentType is not null) + { + Assert.True(conflictingContent.Headers.TryAddWithoutValidation("Content-Type", contentType)); + } + + using HttpResponseMessage rejected = await client.PutAsync(AssignPath, conflictingContent, timeout.Token); + Assert.Equal(HttpStatusCode.UnsupportedMediaType, rejected.StatusCode); + Assert.Same(assigned, manager.State); + + using StringContent replayContent = JsonBody(ValidAssignment); + using HttpResponseMessage replay = await client.PutAsync(AssignPath, replayContent, timeout.Token); + await AssertAssignmentSuccessAsync(replay, HttpStatusCode.NoContent, timeout.Token); + Assert.Same(assigned, manager.State); + } + + [Fact] + public async Task AssignmentBindingUsesRegisteredSourceGeneratedJsonMetadata() + { + await using WorkerProxyWebApplicationFactory factory = new(configureServices: services => + services.PostConfigure(options => + { + Assert.Same(WorkerProxyJsonContext.Default, options.SerializerOptions.TypeInfoResolverChain[0]); + // Retain only the registered context so this request cannot fall back to reflection. + while (options.SerializerOptions.TypeInfoResolverChain.Count > 1) + { + options.SerializerOptions.TypeInfoResolverChain.RemoveAt(1); + } + })); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + manager.OnWorkerAttached(1); + manager.OnWorkerStartStream(1, "worker"); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + using StringContent content = JsonBody(ValidAssignment.Replace( + "startupMode", "STARTUPMODE", StringComparison.Ordinal)); + + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + + await AssertAssignmentSuccessAsync(response, HttpStatusCode.Created, timeout.Token); + Assert.Equal(WorkerStartupMode.SpecializationRequired, manager.State.StartupMode); + Assert.False(manager.State.IsAlwaysReady); } [Theory] [InlineData("application/json", "utf-8")] + [InlineData("application/problem+json", "utf-8")] + [InlineData("APPLICATION/JSON", "utf-8")] [InlineData("application/json; charset=utf-8", "utf-8")] [InlineData("application/json; charset=UTF-8", "utf-8")] [InlineData("application/json; charset=utf-16", "utf-16")] @@ -229,7 +347,6 @@ public async Task SupportedJsonCharsetPreservesAssignment(string contentType, st [InlineData("lastKnownRevision=0%20")] [InlineData("lastKnownRevision=%090")] [InlineData("lastKnownRevision=0%0A")] - [InlineData("lastKnownRevision=0%00")] [InlineData("lastKnownRevision=%000")] [InlineData("lastKnownRevision=0%000")] [InlineData("lastKnownRevision=%C2%A00")] @@ -309,6 +426,7 @@ public async Task ManagementRoutesRejectUnsupportedMethods(string path, string m [InlineData("%2B0")] [InlineData("-0")] [InlineData("000")] + [InlineData("0%00")] public async Task ExplicitZeroRevisionPollReturnsNoContentAtDeadline(string revision) { Mock clock = new(); @@ -390,6 +508,7 @@ public async Task ObsoleteManagementRoutesAreNotFound(string path, string method [InlineData("1")] [InlineData("%2B1")] [InlineData("0001")] + [InlineData("1%00")] public async Task OlderSignedInvariantRevisionReturnsCurrentSnapshot(string revision) { await using WorkerProxyWebApplicationFactory factory = new(); @@ -470,12 +589,14 @@ public async Task ConcurrentIdenticalAssignmentsCreateOnceAndReplaysDoNotChangeR } [Theory] + [InlineData("startupMode", "\"Preconfigured\"", false)] [InlineData("functionAppName", "\"TEST-app\"", false)] [InlineData("functionGroupName", "\"TEST-group\"", false)] [InlineData("functionAppDirectory", "\"/home/site/WWWROOT\"", false)] [InlineData("isAlwaysReady", "true", false)] [InlineData("environment", "{\"SETTING\":\"PRIVATE-value\"}", false)] [InlineData("environment", "{\"setting\":\"private-value\"}", false)] + [InlineData("startupMode", "\"Preconfigured\"", true)] [InlineData("functionAppName", "\"TEST-app\"", true)] [InlineData("functionGroupName", "\"TEST-group\"", true)] [InlineData("functionAppDirectory", "\"/home/site/WWWROOT\"", true)] @@ -529,23 +650,70 @@ public async Task EmptyEnvironmentOrValueIsValidAndNotReadyDoesNotReserveIdentit WorkerPodStateManager manager = factory.Services.GetRequiredService(); using HttpClient client = factory.CreateWorkerProxyClient(); using CancellationTokenSource timeout = new(TestTimeout); + WorkerPodState before = manager.State; using StringContent notReadyContent = JsonBody(ValidAssignment); using HttpResponseMessage notReady = await client.PutAsync(AssignPath, notReadyContent, timeout.Token); await AssertAssignmentErrorAsync(notReady, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); + Assert.Same(before, manager.State); Assert.Equal(0, manager.State.Revision); Assert.Equal(WorkerAssignmentState.Unassigned, manager.State.AssignmentState); Assert.Null(manager.State.FunctionAppName); + Assert.Null(manager.State.StartupMode); Assert.True(manager.OnWorkerAttached(1)); Assert.True(manager.OnWorkerStartStream(1, "worker")); JsonObject assignment = JsonNode.Parse(ValidAssignment)!.AsObject(); assignment["functionAppName"] = "other-app"; + assignment["startupMode"] = "Preconfigured"; assignment["environment"] = JsonNode.Parse(environment); using StringContent content = JsonBody(assignment.ToJsonString()); using HttpResponseMessage created = await client.PutAsync(AssignPath, content, timeout.Token); await AssertAssignmentSuccessAsync(created, HttpStatusCode.Created, timeout.Token); Assert.Equal(3, manager.State.Revision); Assert.Equal("other-app", manager.State.FunctionAppName); + Assert.Equal(WorkerStartupMode.Preconfigured, manager.State.StartupMode); + } + + [Theory] + [InlineData("")] + [InlineData(" \t\r\n")] + public async Task PreconfiguredAssignmentPreservesEmptyOrWhitespaceDirectory(string directory) + { + await using WorkerProxyWebApplicationFactory factory = new(); + WorkerPodStateManager manager = factory.Services.GetRequiredService(); + Assert.True(manager.OnWorkerAttached(1)); + Assert.True(manager.OnWorkerStartStream(1, "worker")); + using HttpClient client = factory.CreateWorkerProxyClient(); + using CancellationTokenSource timeout = new(TestTimeout); + JsonObject body = JsonNode.Parse(ValidAssignment)!.AsObject(); + body["startupMode"] = "Preconfigured"; + body["functionAppDirectory"] = directory; + body["environment"] = new JsonObject(); + using StringContent content = JsonBody(body.ToJsonString()); + using HttpResponseMessage response = await client.PutAsync(AssignPath, content, timeout.Token); + + await AssertAssignmentSuccessAsync(response, HttpStatusCode.Created, timeout.Token); + WorkerPodState assigned = manager.State; + Assert.Equal(WorkerStartupMode.Preconfigured, assigned.StartupMode); + Assert.Equal(WorkerAssignmentState.Ready, assigned.AssignmentState); + Assert.Equal(WorkerPodStatus.ReadyForRequest, assigned.PodStatus); + Assert.Equal(3, assigned.Revision); + + using StringContent replayContent = JsonBody(body.ToJsonString()); + using HttpResponseMessage replay = await client.PutAsync(AssignPath, replayContent, timeout.Token); + await AssertAssignmentSuccessAsync(replay, HttpStatusCode.NoContent, timeout.Token); + Assert.Same(assigned, manager.State); + + body["functionAppDirectory"] = string.Equals(directory, string.Empty, StringComparison.Ordinal) ? " " : string.Empty; + using StringContent changedContent = JsonBody(body.ToJsonString()); + using HttpResponseMessage changed = await client.PutAsync(AssignPath, changedContent, timeout.Token); + await AssertAssignmentErrorAsync(changed, HttpStatusCode.Conflict, "AssignmentConflict", timeout.Token); + Assert.Same(assigned, manager.State); + + using HttpResponseMessage current = await client.GetAsync(StatePath, timeout.Token); + Assert.Equal(HttpStatusCode.OK, current.StatusCode); + using JsonDocument state = JsonDocument.Parse(await current.Content.ReadAsStringAsync(timeout.Token)); + Assert.Equal("Preconfigured", state.RootElement.GetProperty("workerPodState").GetProperty("startupMode").GetString()); } private static void AssertNoStore(HttpResponseMessage response) => diff --git a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs index f2ed360146..d5c3f49a43 100644 --- a/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/ManagementApiHandlersTests.cs @@ -45,18 +45,6 @@ public void GetWorkerReady_TracksStartStreamAndTerminationWithoutChangingState() Assert.Same(terminated, manager.State); } - [Fact] - public void AssignWorker_NullRequestIsInvalidEvenBeforeWorkerReady() - { - WorkerPodStateManager manager = CreateManager(); - WorkerPodState initial = manager.State; - - Assert.Equal(new("InvalidBody", "request"), Assert.Single( - AssertValidation(ManagementApiHandlers.AssignWorker(null, manager)))); - - Assert.Same(initial, manager.State); - } - [Theory] [InlineData(false)] [InlineData(true)] @@ -69,6 +57,7 @@ public void AssignWorker_ReturnsAllMissingFieldsWithoutChangingState(bool ready) Assert.Equal( [ + new("Required", "startupMode"), new("Required", "functionAppName"), new("Required", "functionGroupName"), new("Required", "isAlwaysReady"), @@ -86,6 +75,7 @@ public void AssignWorker_ReturnsAllBlankFieldsAndInvalidEnvironmentTogether() WorkerPodState before = manager.State; WorkerAssignRequest request = new() { + StartupMode = nameof(WorkerStartupMode.SpecializationRequired), FunctionAppName = " ", FunctionGroupName = string.Empty, FunctionAppDirectory = "\t", @@ -129,10 +119,11 @@ public void AssignWorker_InvalidFieldsDoNotClaimAssignment(string field, string? { "app" => CreateRequest(functionAppName: value), "group" => CreateRequest(functionGroupName: value), - "directory" => CreateRequest(functionAppDirectory: value), + "directory" => CreateRequest(functionAppDirectory: value, startupMode: nameof(WorkerStartupMode.SpecializationRequired)), "alwaysReady" => CreateRequest(isAlwaysReady: null), "environment" => new WorkerAssignRequest { + StartupMode = nameof(WorkerStartupMode.Preconfigured), FunctionAppName = "app", FunctionGroupName = "group", FunctionAppDirectory = "app-directory", @@ -195,15 +186,136 @@ public void AssignWorker_EmptyValuesAndNonemptyWhitespaceKeysAreAccepted() CreateRequest(environment: new() { ["SETTING"] = string.Empty, [" "] = string.Empty }), manager)); } + [Theory] + [InlineData(null, "Required")] + [InlineData("", "Required")] + [InlineData(" \t", "Required")] + [InlineData("preconfigured", "InvalidValue")] + [InlineData("specializationrequired", "InvalidValue")] + [InlineData("Preconfigured ", "InvalidValue")] + [InlineData("Unknown", "InvalidValue")] + [InlineData("0", "InvalidValue")] + [InlineData("Preconfigured, SpecializationRequired", "InvalidValue")] + public void AssignWorker_InvalidStartupModeDoesNotClaimOrChangeIdentity(string? startupMode, string code) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest invalid = CreateRequest(startupMode: startupMode); + + Assert.Equal(new(code, "startupMode"), Assert.Single( + AssertValidation(ManagementApiHandlers.AssignWorker(invalid, manager)))); + Assert.Same(before, manager.State); + Assert.Null(manager.State.StartupMode); + + Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); + WorkerPodState assigned = manager.State; + Assert.Equal(new(code, "startupMode"), Assert.Single( + AssertValidation(ManagementApiHandlers.AssignWorker(invalid, manager)))); + Assert.Same(assigned, manager.State); + + manager.OnSessionTerminated(1); + WorkerPodState terminated = manager.State; + Assert.Equal(new(code, "startupMode"), Assert.Single( + AssertValidation(ManagementApiHandlers.AssignWorker(invalid, manager)))); + Assert.Same(terminated, manager.State); + } + [Fact] - public void AssignWorker_RecordsEnvironmentWithoutApplyingItToProxyProcess() + public void AssignWorker_InvalidStartupModeIsCollectedWithOtherFieldErrors() + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest request = new() + { + StartupMode = "private-unsupported-mode", + Environment = new() { ["PRIVATE_SETTING"] = null } + }; + + Assert.Equal( + [ + new("InvalidValue", "startupMode"), + new("Required", "functionAppName"), + new("Required", "functionGroupName"), + new("Required", "isAlwaysReady"), + new("Required", "functionAppDirectory"), + new("InvalidValue", "environment") + ], + AssertValidation(ManagementApiHandlers.AssignWorker(request, manager))); + Assert.Same(before, manager.State); + } + + [Theory] + [InlineData(null, "Required")] + [InlineData("unsupported", "InvalidValue")] + public void AssignWorker_InvalidStartupModeDoesNotSelectDirectoryRules(string? startupMode, string code) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + + IReadOnlyList errors = AssertValidation(ManagementApiHandlers.AssignWorker( + CreateRequest(startupMode: startupMode, functionAppDirectory: string.Empty), manager)); + + Assert.Equal(new(code, "startupMode"), Assert.Single(errors)); + Assert.Same(before, manager.State); + } + + [Theory] + [InlineData("Preconfigured", "")] + [InlineData("Preconfigured", " \t")] + [InlineData("Preconfigured", "/nonexistent-worker-path")] + [InlineData("SpecializationRequired", "/nonexistent-worker-path")] + public void AssignWorker_AcceptsModeDependentDirectoryAndRetainsMode(string startupMode, string directory) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest request = CreateRequest(startupMode: startupMode, functionAppDirectory: directory, environment: new()); + + Assert.IsType(ManagementApiHandlers.AssignWorker(request, manager)); + WorkerPodState assigned = manager.State; + Assert.Equal(startupMode, assigned.StartupMode.ToString()); + Assert.Equal(before.Revision + 1, assigned.Revision); + Assert.Equal(WorkerAssignmentState.Ready, assigned.AssignmentState); + Assert.Equal(WorkerPodStatus.ReadyForRequest, assigned.PodStatus); + Assert.IsType(ManagementApiHandlers.AssignWorker(request, manager)); + Assert.Same(assigned, manager.State); + AssertError(ManagementApiHandlers.AssignWorker( + CreateRequest(startupMode: startupMode, functionAppDirectory: directory + "/changed", environment: new()), manager), + 409, "AssignmentConflict"); + Assert.Same(assigned, manager.State); + } + + [Theory] + [InlineData("Preconfigured")] + [InlineData("SpecializationRequired")] + public void AssignWorker_BothModesRequireEnvironmentAndDirectory(string startupMode) + { + WorkerPodStateManager manager = CreateReadyManager(); + WorkerPodState before = manager.State; + WorkerAssignRequest request = new() + { + StartupMode = startupMode, + FunctionAppName = "app", + FunctionGroupName = "group", + IsAlwaysReady = false + }; + + Assert.Equal( + [new("Required", "functionAppDirectory"), new("Required", "environment")], + AssertValidation(ManagementApiHandlers.AssignWorker(request, manager))); + Assert.Same(before, manager.State); + } + + [Theory] + [InlineData("Preconfigured")] + [InlineData("SpecializationRequired")] + public void AssignWorker_RecordsEnvironmentWithoutApplyingItToProxyProcess(string startupMode) { WorkerPodStateManager manager = CreateReadyManager(); string setting = $"WORKERPROXY_ASSIGNMENT_TEST_{Guid.NewGuid():N}"; Assert.Null(Environment.GetEnvironmentVariable(setting)); Assert.IsType(ManagementApiHandlers.AssignWorker( - CreateRequest(environment: new() { [setting] = "private-value" }), manager)); + CreateRequest(startupMode: startupMode, environment: new() { [setting] = "private-value" }), manager)); Assert.Null(Environment.GetEnvironmentVariable(setting)); Assert.Equal(WorkerAssignmentState.Ready, manager.State.AssignmentState); @@ -263,12 +375,14 @@ public void AssignWorker_CopiesEnvironmentAndReplaysEquivalentIdentityRegardless [InlineData("alwaysReady")] [InlineData("environmentKey")] [InlineData("environmentValue")] + [InlineData("startupMode")] public void AssignWorker_DifferentIdentityConflictsBeforeAndAfterTermination(string field) { WorkerPodStateManager manager = CreateReadyManager(); Assert.IsType(ManagementApiHandlers.AssignWorker(CreateRequest(), manager)); WorkerAssignRequest different = field switch { + "startupMode" => CreateRequest(startupMode: nameof(WorkerStartupMode.SpecializationRequired)), "app" => CreateRequest(functionAppName: "APP"), "group" => CreateRequest(functionGroupName: "GROUP"), "directory" => CreateRequest(functionAppDirectory: "APP-DIRECTORY"), @@ -286,6 +400,7 @@ public void AssignWorker_DifferentIdentityConflictsBeforeAndAfterTermination(str AssertError(ManagementApiHandlers.AssignWorker(different, manager), 409, "AssignmentConflict"); AssertError(ManagementApiHandlers.AssignWorker(CreateRequest(), manager), 409, "WorkerTerminated"); + Assert.Equal(WorkerStartupMode.Preconfigured, terminated.StartupMode); Assert.Same(terminated, manager.State); } @@ -520,9 +635,11 @@ private static WorkerAssignRequest CreateRequest( string? functionGroupName = "group", bool? isAlwaysReady = false, string? functionAppDirectory = "app-directory", - Dictionary? environment = null) => + Dictionary? environment = null, + string? startupMode = nameof(WorkerStartupMode.Preconfigured)) => new() { + StartupMode = startupMode, FunctionAppName = functionAppName, FunctionGroupName = functionGroupName, IsAlwaysReady = isAlwaysReady, diff --git a/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs index 1e23047855..2f3122b831 100644 --- a/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs +++ b/test/Functions.WorkerProxy.Tests/Management/WorkerProxyJsonContextTests.cs @@ -12,11 +12,14 @@ namespace Azure.Functions.WorkerProxy.Tests.Management; public class WorkerProxyJsonContextTests { - [Fact] - public void Assignment_RoundTripsWithCamelCaseAndExplicitFalse() + [Theory] + [InlineData("Preconfigured")] + [InlineData("SpecializationRequired")] + public void Assignment_RoundTripsWithCamelCaseAndExplicitFalse(string startupMode) { WorkerAssignRequest request = new() { + StartupMode = startupMode, FunctionAppName = "app", FunctionGroupName = "group", FunctionAppDirectory = "private-directory", @@ -27,13 +30,15 @@ public void Assignment_RoundTripsWithCamelCaseAndExplicitFalse() string json = JsonSerializer.Serialize(request, WorkerProxyJsonContext.Default.WorkerAssignRequest); using JsonDocument document = JsonDocument.Parse(json); JsonElement root = document.RootElement; - AssertProperties(root, "functionAppName", "functionGroupName", "functionAppDirectory", "isAlwaysReady", "environment"); + AssertProperties(root, "startupMode", "functionAppName", "functionGroupName", "functionAppDirectory", "isAlwaysReady", "environment"); + Assert.Equal(startupMode, root.GetProperty("startupMode").GetString()); Assert.False(root.GetProperty("isAlwaysReady").GetBoolean()); Assert.Equal(string.Empty, root.GetProperty("environment").GetProperty("MixedCase_SETTING").GetString()); Assert.Equal("private-value", root.GetProperty("environment").GetProperty("SECRET").GetString()); WorkerAssignRequest copy = Assert.IsType( JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + Assert.Equal(request.StartupMode, copy.StartupMode); Assert.Equal(request.FunctionAppName, copy.FunctionAppName); Assert.Equal(request.FunctionGroupName, copy.FunctionGroupName); Assert.Equal(request.FunctionAppDirectory, copy.FunctionAppDirectory); @@ -45,13 +50,14 @@ public void Assignment_RoundTripsWithCamelCaseAndExplicitFalse() public void Assignment_PropertyNamesAreCaseInsensitiveButEnvironmentKeysArePreserved() { const string json = """ - {"FUNCTIONAPPNAME":"app","FunctionGroupName":"group","FUNCTIONAPPDIRECTORY":"directory", + {"STARTUPMODE":"Preconfigured","FUNCTIONAPPNAME":"app","FunctionGroupName":"group","FUNCTIONAPPDIRECTORY":"directory", "ISALWAYSREADY":true,"ENVIRONMENT":{"Key":"one","key":"two"}} """; WorkerAssignRequest request = Assert.IsType( JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + Assert.Equal("Preconfigured", request.StartupMode); Assert.Equal("app", request.FunctionAppName); Assert.Equal("group", request.FunctionGroupName); Assert.Equal("directory", request.FunctionAppDirectory); @@ -63,12 +69,13 @@ public void Assignment_PropertyNamesAreCaseInsensitiveButEnvironmentKeysArePrese [Theory] [InlineData("{}")] - [InlineData("""{"functionAppName":null,"functionGroupName":null,"functionAppDirectory":null,"isAlwaysReady":null,"environment":null}""")] + [InlineData("""{"startupMode":null,"functionAppName":null,"functionGroupName":null,"functionAppDirectory":null,"isAlwaysReady":null,"environment":null}""")] public void Assignment_MissingRequiredValuesAreLeftForHandlerValidation(string json) { WorkerAssignRequest request = Assert.IsType( JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + Assert.Null(request.StartupMode); Assert.Null(request.FunctionAppName); Assert.Null(request.FunctionGroupName); Assert.Null(request.FunctionAppDirectory); @@ -88,6 +95,10 @@ public void Assignment_NullEnvironmentValueIsLeftForHandlerValidation() } [Theory] + [InlineData("""{"startupMode":0}""")] + [InlineData("""{"startupMode":true}""")] + [InlineData("""{"startupMode":[]}""")] + [InlineData("""{"startupMode":{}}""")] [InlineData("""{"isAlwaysReady":"false"}""")] [InlineData("""{"isAlwaysReady":0}""")] [InlineData("""{"functionAppName":123}""")] @@ -103,6 +114,22 @@ public void Assignment_MalformedTypesThrowJsonException(string json) JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); } + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("preconfigured")] + [InlineData("Unknown")] + [InlineData("0")] + public void Assignment_InvalidModeStringsArePreservedForFieldValidation(string startupMode) + { + string json = $$"""{"startupMode":"{{startupMode}}"}"""; + + WorkerAssignRequest request = Assert.IsType( + JsonSerializer.Deserialize(json, WorkerProxyJsonContext.Default.WorkerAssignRequest)); + + Assert.Equal(startupMode, request.StartupMode); + } + [Fact] public void NullAssignment_DeserializesAsNullForHandlerValidation() { @@ -128,10 +155,13 @@ public void InitialState_ContainsOnlyPublicShapeAndOmitsUnknownIdentity() } [Theory] - [InlineData(false)] - [InlineData(true)] - public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool isAlwaysReady) + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool isAlwaysReady, bool specializationRequired) { + WorkerStartupMode startupMode = specializationRequired ? WorkerStartupMode.SpecializationRequired : WorkerStartupMode.Preconfigured; WorkerPodState state = new( PodName: "pod", Revision: long.MaxValue, @@ -139,6 +169,7 @@ public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool IsWorkerAttached: true, WorkerId: "private-worker", AssignmentState: WorkerAssignmentState.Ready, + StartupMode: startupMode, FunctionAppName: "private-app", FunctionGroupName: "group", IsAlwaysReady: isAlwaysReady); @@ -154,7 +185,8 @@ public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool Assert.Equal("FunctionsWorkerPod", root.GetProperty("functionsContainerType").GetString()); Assert.Equal(JsonValueKind.Number, root.GetProperty("revisionId").ValueKind); Assert.Equal(long.MaxValue, root.GetProperty("revisionId").GetInt64()); - AssertProperties(podState, "podStatus", "functionGroupName", "isAlwaysReady"); + AssertProperties(podState, "podStatus", "startupMode", "functionGroupName", "isAlwaysReady"); + Assert.Equal(startupMode.ToString(), podState.GetProperty("startupMode").GetString()); Assert.Equal("ReadyForRequest", podState.GetProperty("podStatus").GetString()); Assert.Equal("group", podState.GetProperty("functionGroupName").GetString()); Assert.Equal(isAlwaysReady, podState.GetProperty("isAlwaysReady").GetBoolean()); @@ -166,7 +198,7 @@ public void AssignedState_ExposesOnlyPublicIdentityAndPreservesLongRevision(bool [Fact] public void FromState_RetainedResponseDoesNotChangeWhenSourceStateIsReplaced() { - WorkerPodState ready = new("pod", 3, 1, true, "worker", WorkerAssignmentState.Ready, "app", "group", false); + WorkerPodState ready = new("pod", 3, 1, true, "worker", WorkerAssignmentState.Ready, WorkerStartupMode.Preconfigured, "app", "group", false); WorkerInstanceState retained = WorkerInstanceState.FromState(ready); WorkerPodState terminated = ready with { @@ -186,6 +218,7 @@ public void FromState_RetainedResponseDoesNotChangeWhenSourceStateIsReplaced() Assert.Equal(4, terminatedDocument.RootElement.GetProperty("revisionId").GetInt64()); JsonElement terminatedPodState = terminatedDocument.RootElement.GetProperty("workerPodState"); Assert.Equal("None", terminatedPodState.GetProperty("podStatus").GetString()); + Assert.Equal("Preconfigured", terminatedPodState.GetProperty("startupMode").GetString()); Assert.Equal("group", terminatedPodState.GetProperty("functionGroupName").GetString()); Assert.False(terminatedPodState.GetProperty("isAlwaysReady").GetBoolean()); } @@ -246,7 +279,7 @@ public void ValidationEnvelope_RoundTripsAllFieldErrorsUsingGeneratedMetadata() [Fact] public void Context_RecursivelyGeneratesMetadataForNestedResponseTypes() { - WorkerPodStateResponse state = new(WorkerPodStatus.ReadyForRequest, "group", false); + WorkerPodStateResponse state = new(WorkerPodStatus.ReadyForRequest, WorkerStartupMode.Preconfigured, "group", false); WorkerApiError error = new("WorkerNotReady"); string stateJson = JsonSerializer.Serialize(state, WorkerProxyJsonContext.Default.WorkerPodStateResponse); @@ -256,6 +289,7 @@ public void Context_RecursivelyGeneratesMetadataForNestedResponseTypes() Assert.Equal(error, JsonSerializer.Deserialize(errorJson, WorkerProxyJsonContext.Default.WorkerApiError)); using JsonDocument stateDocument = JsonDocument.Parse(stateJson); Assert.Equal("ReadyForRequest", stateDocument.RootElement.GetProperty("podStatus").GetString()); + Assert.Equal("Preconfigured", stateDocument.RootElement.GetProperty("startupMode").GetString()); Assert.False(stateDocument.RootElement.GetProperty("isAlwaysReady").GetBoolean()); } diff --git a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs index 454ccfc92b..760e671357 100644 --- a/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs +++ b/test/Functions.WorkerProxy.Tests/Rpc/FunctionRpcRelayTests.Management.cs @@ -21,15 +21,18 @@ namespace Azure.Functions.WorkerProxy.Tests; public partial class FunctionRpcRelayTests { private const string ManagementAssignment = """ - {"functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, + {"startupMode":"SpecializationRequired","functionAppName":"test-app","functionGroupName":"test-group","isAlwaysReady":false, "environment":{"B":"private-value","A":"1"},"functionAppDirectory":"/home/site/wwwroot"} """; [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task ManagementApis_ObserveNormalStartupAssignmentAndTerminalFailure(bool runtimeFirst) + [InlineData(true, "Preconfigured")] + [InlineData(false, "Preconfigured")] + [InlineData(true, "SpecializationRequired")] + [InlineData(false, "SpecializationRequired")] + public async Task ManagementApis_ObserveNormalStartupAssignmentAndTerminalFailure(bool runtimeFirst, string startupMode) { + string assignmentBody = ManagementAssignment.Replace("SpecializationRequired", startupMode, StringComparison.Ordinal); const string proxyEndpoint = "http://worker-pod:28080/"; await using WorkerProxyWebApplicationFactory factory = CreateHttpCapabilityFactory(proxyEndpoint); WorkerPodStateManager manager = factory.Services.GetRequiredService(); @@ -38,7 +41,7 @@ public async Task ManagementApis_ObserveNormalStartupAssignmentAndTerminalFailur await AssertReadinessAsync(management, "/admin/instance/ready", HttpStatusCode.OK, timeout.Token); await AssertReadinessAsync(management, "/admin/worker/ready", HttpStatusCode.ServiceUnavailable, timeout.Token); using (HttpResponseMessage notReady = await PutManagementJsonAsync( - management, "/admin/worker/assignment", ManagementAssignment, timeout.Token)) + management, "/admin/worker/assignment", assignmentBody, timeout.Token)) { await AssertManagementErrorAsync(notReady, HttpStatusCode.ServiceUnavailable, "WorkerNotReady", timeout.Token); } @@ -74,13 +77,14 @@ await WaitForAttachmentAsync(factory.Services.GetRequiredService assignmentPoll = management.GetAsync( "/admin/worker/state?lastKnownRevision=2", timeout.Token); await WaitForManagementPollAsync(manager, timeout.Token); using (HttpResponseMessage assignment = await PutManagementJsonAsync( - management, "/admin/worker/assignment", ManagementAssignment, timeout.Token)) + management, "/admin/worker/assignment", assignmentBody, timeout.Token)) { Assert.Equal(HttpStatusCode.Created, assignment.StatusCode); Assert.Equal("/admin/worker/assignment", assignment.Headers.Location?.OriginalString); @@ -98,6 +102,7 @@ await WaitForAttachmentAsync(factory.Services.GetRequiredService new("test-app", "test-group", isAlwaysReady: false, + => new(WorkerStartupMode.SpecializationRequired, "test-app", "test-group", isAlwaysReady: false, environment: new Dictionary(), functionAppDirectory: "/home/site/wwwroot"); private sealed class PumpingSynchronizationContext : SynchronizationContext diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs index e48364b8d6..4e5d41a5f7 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerAssignmentTests.cs @@ -34,6 +34,7 @@ public void Equality_IgnoresEnvironmentOrderAndCopiesInput() [InlineData("key")] [InlineData("value")] [InlineData("count")] + [InlineData("startupMode")] public void Equality_UsesEveryFieldAndOrdinalStrings(string changedField) { WorkerAssignment original = Create(new Dictionary { ["KEY"] = "Value" }); @@ -48,6 +49,8 @@ public void Equality_UsesEveryFieldAndOrdinalStrings(string changedField) } WorkerAssignment changed = new( + string.Equals(changedField, "startupMode", StringComparison.Ordinal) + ? WorkerStartupMode.SpecializationRequired : WorkerStartupMode.Preconfigured, string.Equals(changedField, "app", StringComparison.Ordinal) ? "APP" : "app", string.Equals(changedField, "group", StringComparison.Ordinal) ? "HTTP" : "http", string.Equals(changedField, "alwaysReady", StringComparison.Ordinal), @@ -75,9 +78,50 @@ public void Equality_PreservesDistinctEnvironmentKeyCasing() public void Construction_RejectsMissingIdentity(string? value) { Dictionary environment = []; - Assert.ThrowsAny(() => new WorkerAssignment(value!, "http", false, environment, "/app")); - Assert.ThrowsAny(() => new WorkerAssignment("app", value!, false, environment, "/app")); - Assert.ThrowsAny(() => new WorkerAssignment("app", "http", false, environment, value!)); + Assert.ThrowsAny(() => new WorkerAssignment(WorkerStartupMode.Preconfigured, value!, "http", false, environment, "/app")); + Assert.ThrowsAny(() => new WorkerAssignment(WorkerStartupMode.Preconfigured, "app", value!, false, environment, "/app")); + Assert.ThrowsAny(() => new WorkerAssignment(WorkerStartupMode.SpecializationRequired, "app", "http", false, environment, value!)); + } + + [Theory] + [InlineData("")] + [InlineData(" \t")] + [InlineData("/app")] + public void Construction_PreconfiguredPreservesDirectoryWithoutApplyingValidationForSpecialization(string directory) + { + WorkerAssignment assignment = new(WorkerStartupMode.Preconfigured, "app", "http", false, new Dictionary(), directory); + + Assert.Equal(WorkerStartupMode.Preconfigured, assignment.StartupMode); + Assert.Equal(directory, assignment.FunctionAppDirectory); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Construction_RejectsNullDirectoryInBothModes(bool specializationRequired) + { + WorkerStartupMode startupMode = specializationRequired ? WorkerStartupMode.SpecializationRequired : WorkerStartupMode.Preconfigured; + Assert.Throws(() => + new WorkerAssignment(startupMode, "app", "http", false, new Dictionary(), null!)); + } + + [Theory] + [InlineData(-1)] + [InlineData(2)] + public void Construction_RejectsUndefinedStartupMode(int startupMode) + { + Assert.Throws(() => + new WorkerAssignment((WorkerStartupMode)startupMode, "app", "http", false, new Dictionary(), "/app")); + } + + [Fact] + public void Equality_PreconfiguredEmptyDirectoryStillParticipatesInIdentity() + { + WorkerAssignment empty = new(WorkerStartupMode.Preconfigured, "app", "http", false, new Dictionary(), string.Empty); + WorkerAssignment whitespace = new(WorkerStartupMode.Preconfigured, "app", "http", false, new Dictionary(), " "); + + Assert.False(empty.IsEquivalentTo(whitespace)); + Assert.False(whitespace.IsEquivalentTo(empty)); } [Fact] @@ -95,5 +139,5 @@ public void Equality_RejectsNull() } private static WorkerAssignment Create(IReadOnlyDictionary environment) - => new("app", "http", false, environment, "/home/site/wwwroot"); + => new(WorkerStartupMode.Preconfigured, "app", "http", false, environment, "/home/site/wwwroot"); } diff --git a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs index 2da8597919..66475b742f 100644 --- a/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs +++ b/test/Functions.WorkerProxy.Tests/State/WorkerPodStateManagerTests.cs @@ -26,6 +26,7 @@ public void InitialState_IsUnassignedAtRevisionZero() Assert.False(state.IsWorkerReady); Assert.Null(state.WorkerId); Assert.Equal(WorkerAssignmentState.Unassigned, state.AssignmentState); + Assert.Null(state.StartupMode); Assert.Equal(WorkerPodStatus.None, state.PodStatus); Assert.Null(state.FunctionAppName); Assert.Null(state.FunctionGroupName); @@ -33,9 +34,12 @@ public void InitialState_IsUnassignedAtRevisionZero() Assert.Same(state, manager.State); } - [Fact] - public void Lifecycle_PublishesImmutableMonotonicSnapshots() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Lifecycle_PublishesImmutableMonotonicSnapshots(bool specializationRequired) { + WorkerStartupMode startupMode = specializationRequired ? WorkerStartupMode.SpecializationRequired : WorkerStartupMode.Preconfigured; WorkerPodStateManager manager = new(CreateOptions(), TimeProvider.System); WorkerPodState initial = manager.State; Assert.True(manager.OnWorkerAttached(1)); @@ -46,9 +50,10 @@ public void Lifecycle_PublishesImmutableMonotonicSnapshots() Assert.True(started.IsWorkerReady); Assert.Equal(WorkerPodStatus.None, started.PodStatus); - Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment())); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment(startupMode: startupMode))); WorkerPodState assigned = manager.State; Assert.Equal(WorkerAssignmentState.Ready, assigned.AssignmentState); + Assert.Equal(startupMode, assigned.StartupMode); Assert.Equal(WorkerPodStatus.ReadyForRequest, assigned.PodStatus); Assert.Equal("app", assigned.FunctionAppName); Assert.Equal("http", assigned.FunctionGroupName); @@ -61,6 +66,7 @@ public void Lifecycle_PublishesImmutableMonotonicSnapshots() Assert.False(failed.IsWorkerReady); Assert.False(failed.IsWorkerAttached); Assert.Equal(WorkerAssignmentState.Failed, failed.AssignmentState); + Assert.Equal(startupMode, failed.StartupMode); Assert.Equal(WorkerPodStatus.None, failed.PodStatus); Assert.Equal("app", failed.FunctionAppName); Assert.Equal(new long[] { 0, 1, 2, 3, 4 }, @@ -69,6 +75,7 @@ public void Lifecycle_PublishesImmutableMonotonicSnapshots() Assert.False(initial.IsWorkerAttached); Assert.Null(attached.WorkerId); Assert.Equal(WorkerAssignmentState.Unassigned, started.AssignmentState); + Assert.Null(started.StartupMode); Assert.True(assigned.IsWorkerReady); } @@ -79,6 +86,7 @@ public void NotReadyAssignment_DoesNotClaimIdentityOrChangeRevision() WorkerPodState initial = manager.State; Assert.Equal(WorkerAssignmentResult.WorkerNotReady, manager.Assign(CreateAssignment("rejected"))); Assert.Same(initial, manager.State); + Assert.Null(manager.State.StartupMode); manager.OnWorkerAttached(1); WorkerPodState attached = manager.State; @@ -86,8 +94,9 @@ public void NotReadyAssignment_DoesNotClaimIdentityOrChangeRevision() Assert.Same(attached, manager.State); manager.OnWorkerStartStream(1, "worker"); - Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment("accepted"))); + Assert.Equal(WorkerAssignmentResult.Created, manager.Assign(CreateAssignment("accepted", WorkerStartupMode.SpecializationRequired))); Assert.Equal("accepted", manager.State.FunctionAppName); + Assert.Equal(WorkerStartupMode.SpecializationRequired, manager.State.StartupMode); } [Fact] @@ -286,6 +295,30 @@ public async Task ConcurrentConflictingAssignments_HaveOneWinningIdentity() Assert.Equal(3, manager.State.Revision); } + [Fact] + public async Task ConcurrentStartupModes_ClaimOneImmutableMode() + { + WorkerPodStateManager manager = CreateReadyManager(); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task<(WorkerStartupMode Mode, WorkerAssignmentResult Result)>[] attempts = Enumerable.Range(0, 32).Select(index => Task.Run(async () => + { + await start.Task; + WorkerStartupMode mode = index % 2 == 0 ? WorkerStartupMode.Preconfigured : WorkerStartupMode.SpecializationRequired; + return (mode, manager.Assign(CreateAssignment(startupMode: mode))); + })).ToArray(); + + start.SetResult(); + (WorkerStartupMode Mode, WorkerAssignmentResult Result)[] results = await Task.WhenAll(attempts).WaitAsync(TestTimeout); + (WorkerStartupMode Mode, WorkerAssignmentResult Result) winner = Assert.Single(results, result => result.Result == WorkerAssignmentResult.Created); + + Assert.Equal(winner.Mode, manager.State.StartupMode); + Assert.Equal(15, results.Count(result => result.Result == WorkerAssignmentResult.AlreadyAssigned)); + Assert.Equal(16, results.Count(result => result.Result == WorkerAssignmentResult.AssignmentConflict)); + Assert.All(results.Where(result => result.Mode != winner.Mode), + result => Assert.Equal(WorkerAssignmentResult.AssignmentConflict, result.Result)); + Assert.Equal(3, manager.State.Revision); + } + [Fact] public async Task AssignmentRacingTermination_CannotLeaveReadyState() { @@ -333,8 +366,8 @@ private static WorkerPodStateManager CreateReadyManager() return manager; } - private static WorkerAssignment CreateAssignment(string appName = "app") - => new(appName, "http", false, new Dictionary { ["SETTING"] = "value" }, "/home/site/wwwroot"); + private static WorkerAssignment CreateAssignment(string appName = "app", WorkerStartupMode startupMode = WorkerStartupMode.Preconfigured) + => new(startupMode, appName, "http", false, new Dictionary { ["SETTING"] = "value" }, "/home/site/wwwroot"); private static IOptions CreateOptions(string podName = "pod") => Options.Create(new WorkerProxyOptions { PodName = podName }); From 3644e34955da2a976aed71d966c8cba58f9bf24c Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Mon, 14 Sep 2026 22:18:08 -0700 Subject: [PATCH 10/10] Clarify WorkerProxy management API error documentation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Management/ManagementApiEndpoints.cs | 8 +++++--- .../Management/WorkerApiErrorResponse.cs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs index 992a97bf10..47aa79f448 100644 --- a/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs +++ b/src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs @@ -13,9 +13,11 @@ namespace Azure.Functions.WorkerProxy.Management; /// Registers worker lifecycle APIs on the management listener. /// /// -/// Platform callers are expected to send UTF-8 JSON. Assignment uses framework JSON binding; -/// binding failures follow framework behavior without a guaranteed status or error envelope. -/// Successfully bound requests use our field-validation envelope and lifecycle error codes. +/// Platform callers are expected to send UTF-8 JSON. Assignment uses framework JSON binding: +/// malformed or incompatible JSON with a supported charset returns HTTP 400; +/// unsupported media types return HTTP 415. Binding errors do not guarantee our validation envelope. +/// Other binding failures follow framework behavior. Successfully bound requests use our +/// field-validation envelope and lifecycle error codes. /// internal static class ManagementApiEndpoints { diff --git a/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs b/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs index c5429deab7..f4d5034475 100644 --- a/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs +++ b/src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs @@ -4,6 +4,6 @@ namespace Azure.Functions.WorkerProxy.Management; /// -/// Wraps validation and lifecycle failures in the same management API error envelope. +/// Wraps worker lifecycle failures in the management API error envelope. /// internal sealed record WorkerApiErrorResponse(WorkerApiError Error);