I have an ASP.NET Core app with a BackgroundService (IHostedService). Behavior:
Goal: When doing a deployment, slot swap, or stopping the App Service in Azure, I want the message to finish processing (not abruptly fail) before the app is completely shut down.
What appears to happen is regardless of the code I write in Program.cs or what environment variables I set on the App Service, the App will forcefully shut down anywhere between 5-15 seconds without finishing the StopAsync() method (during ProcessMessageAsync(). Here is an example of the log stream I see:
Notice how the logging messages from StopAsync() are not logged because await _serviceBusProcessor.StopProcessingAsync() is blocking and never finishes. I expect the message to finish all 20 loops and the rest of StopAsync() to execute.
When I run the app locally in Visual Studio, the app delays shutdown until the message is finished processing with the correct outputs when I enter Ctrl + C:
My code:
Program.cs:
public static void Main(string[] args)
{
var builder = WebApplication.CreateBuilder(args);
builder.Host.ConfigureHostOptions(options => options.ShutdownTimeout = TimeSpan.FromSeconds(60));
builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(60));
builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();
builder.Services.AddHostedService<ServiceBusHostedService>();
var app = builder.Build();
app.UseStaticFiles();
app.UseSwagger();
app.UseSwaggerUI(options =>
{
options.EnableTryItOutByDefault();
});
app.UseHttpsRedirection();
app.UseAuthorization();
app.MapControllers();
app.Run();
}
BackgroundService:
using Azure.Identity;
using Azure.Messaging.ServiceBus;
namespace MyApp.API.ServiceBus
{
public class ServiceBusHostedService : BackgroundService
{
private readonly ILogger<ServiceBusHostedService> _logger;
private readonly ServiceBusClient _serviceBusClient;
private readonly ServiceBusProcessor _serviceBusProcessor;
public ServiceBusHostedService(
ILogger<ServiceBusHostedService> logger,
ServiceBusConfiguration serviceBusConfiguration)
{
_logger = logger;
_serviceBusClient = new ServiceBusClient(serviceBusConfiguration.Namespace, new DefaultAzureCredential());
_serviceBusProcessor = _serviceBusClient.CreateProcessor(serviceBusConfiguration.WorkQueueName, new ServiceBusProcessorOptions
{
MaxConcurrentCalls = serviceBusConfiguration.NumberOfThreads,
AutoCompleteMessages = false,
});
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("ServiceBusHostedService started");
_serviceBusProcessor.ProcessMessageAsync += ProcessMessageAsync;
_serviceBusProcessor.ProcessErrorAsync += ProcessErrorAsync;
await _serviceBusProcessor.StartProcessingAsync(stoppingToken);
await Task.Delay(Timeout.Infinite, stoppingToken);
}
private async Task ProcessMessageAsync(ProcessMessageEventArgs args)
{
var message = args.Message;
var sequenceNumber = message.SequenceNumber.ToString();
try
{
var messageTypeExists = message.ApplicationProperties.TryGetValue("MessageType", out var messageType);
// Let's make this take about 20 seconds to process
for (var i = 0; i < 20; i++)
{
_logger.LogInformation($"Sleeping... i = {i}");
Thread.Sleep(1000);
}
_logger.LogWarning(message.Body.ToString());
await args.CompleteMessageAsync(message);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error while processing message with sequence number {sequenceNumber} for {args.FullyQualifiedNamespace} and entity path {args.EntityPath}");
try
{
await args.AbandonMessageAsync(args.Message);
}
catch (Exception abandonEx)
{
_logger.LogCritical(abandonEx, "Failed to abandon message");
}
}
}
private Task ProcessErrorAsync(ProcessErrorEventArgs args)
{
_logger.LogError($"Error in Service Bus Processor for {args.FullyQualifiedNamespace} at {args.ErrorSource} for entity path {args.EntityPath}: {args.Exception}");
return Task.CompletedTask;
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation($"{nameof(ServiceBusHostedService)} is stopping");
await _serviceBusProcessor.StopProcessingAsync();
// Wait for ongoing message processing to complete.
while (_serviceBusProcessor.IsProcessing)
{
_logger.LogInformation("Waiting for ongoing message processing to complete..");
if (cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("Shutdown was forcibly canceled");
break;
}
await Task.Delay(1000, cancellationToken);
}
_logger.LogInformation("All message processing completed. Cleaning up resources");
// Dispose of resources.
await _serviceBusProcessor.DisposeAsync();
await _serviceBusClient.DisposeAsync();
await base.StopAsync(cancellationToken);
_logger.LogInformation($"{nameof(ServiceBusHostedService)} stopped gracefully");
}
}
}
I've also tried setting the following environment variables on the App Service, neither of which had any impact:
ASPNETCORE_SHUTDOWNTIMEOUTSECONDS = 60WEBSITE_WAIT_FOR_SHUTDOWN = 60To summarize, how can I get the Azure App Service to terminate gracefully with the extended timeout that I am able to achieve when running the app locally from Visual Studio?
This is the only helpful link I could find in any Microsoft doc, but perhaps it doesn't apply to Azure App Services: .0
Any help would be much appreciated!
I have an ASP.NET Core app with a BackgroundService (IHostedService). Behavior:
Goal: When doing a deployment, slot swap, or stopping the App Service in Azure, I want the message to finish processing (not abruptly fail) before the app is completely shut down.
What appears to happen is regardless of the code I write in Program.cs or what environment variables I set on the App Service, the App will forcefully shut down anywhere between 5-15 seconds without finishing the StopAsync() method (during ProcessMessageAsync(). Here is an example of the log stream I see:
Notice how the logging messages from StopAsync() are not logged because await _serviceBusProcessor.StopProcessingAsync() is blocking and never finishes. I expect the message to finish all 20 loops and the rest of StopAsync() to execute.
When I run the app locally in Visual Studio, the app delays shutdown until the message is finished processing with the correct outputs when I enter Ctrl + C:
My code:
Program.cs:
public static void Main(string[] args)
{
var builder = WebApplication.CreateBuilder(args);
builder.Host.ConfigureHostOptions(options => options.ShutdownTimeout = TimeSpan.FromSeconds(60));
builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(60));
builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();
builder.Services.AddHostedService<ServiceBusHostedService>();
var app = builder.Build();
app.UseStaticFiles();
app.UseSwagger();
app.UseSwaggerUI(options =>
{
options.EnableTryItOutByDefault();
});
app.UseHttpsRedirection();
app.UseAuthorization();
app.MapControllers();
app.Run();
}
BackgroundService:
using Azure.Identity;
using Azure.Messaging.ServiceBus;
namespace MyApp.API.ServiceBus
{
public class ServiceBusHostedService : BackgroundService
{
private readonly ILogger<ServiceBusHostedService> _logger;
private readonly ServiceBusClient _serviceBusClient;
private readonly ServiceBusProcessor _serviceBusProcessor;
public ServiceBusHostedService(
ILogger<ServiceBusHostedService> logger,
ServiceBusConfiguration serviceBusConfiguration)
{
_logger = logger;
_serviceBusClient = new ServiceBusClient(serviceBusConfiguration.Namespace, new DefaultAzureCredential());
_serviceBusProcessor = _serviceBusClient.CreateProcessor(serviceBusConfiguration.WorkQueueName, new ServiceBusProcessorOptions
{
MaxConcurrentCalls = serviceBusConfiguration.NumberOfThreads,
AutoCompleteMessages = false,
});
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("ServiceBusHostedService started");
_serviceBusProcessor.ProcessMessageAsync += ProcessMessageAsync;
_serviceBusProcessor.ProcessErrorAsync += ProcessErrorAsync;
await _serviceBusProcessor.StartProcessingAsync(stoppingToken);
await Task.Delay(Timeout.Infinite, stoppingToken);
}
private async Task ProcessMessageAsync(ProcessMessageEventArgs args)
{
var message = args.Message;
var sequenceNumber = message.SequenceNumber.ToString();
try
{
var messageTypeExists = message.ApplicationProperties.TryGetValue("MessageType", out var messageType);
// Let's make this take about 20 seconds to process
for (var i = 0; i < 20; i++)
{
_logger.LogInformation($"Sleeping... i = {i}");
Thread.Sleep(1000);
}
_logger.LogWarning(message.Body.ToString());
await args.CompleteMessageAsync(message);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error while processing message with sequence number {sequenceNumber} for {args.FullyQualifiedNamespace} and entity path {args.EntityPath}");
try
{
await args.AbandonMessageAsync(args.Message);
}
catch (Exception abandonEx)
{
_logger.LogCritical(abandonEx, "Failed to abandon message");
}
}
}
private Task ProcessErrorAsync(ProcessErrorEventArgs args)
{
_logger.LogError($"Error in Service Bus Processor for {args.FullyQualifiedNamespace} at {args.ErrorSource} for entity path {args.EntityPath}: {args.Exception}");
return Task.CompletedTask;
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation($"{nameof(ServiceBusHostedService)} is stopping");
await _serviceBusProcessor.StopProcessingAsync();
// Wait for ongoing message processing to complete.
while (_serviceBusProcessor.IsProcessing)
{
_logger.LogInformation("Waiting for ongoing message processing to complete..");
if (cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("Shutdown was forcibly canceled");
break;
}
await Task.Delay(1000, cancellationToken);
}
_logger.LogInformation("All message processing completed. Cleaning up resources");
// Dispose of resources.
await _serviceBusProcessor.DisposeAsync();
await _serviceBusClient.DisposeAsync();
await base.StopAsync(cancellationToken);
_logger.LogInformation($"{nameof(ServiceBusHostedService)} stopped gracefully");
}
}
}
I've also tried setting the following environment variables on the App Service, neither of which had any impact:
ASPNETCORE_SHUTDOWNTIMEOUTSECONDS = 60WEBSITE_WAIT_FOR_SHUTDOWN = 60To summarize, how can I get the Azure App Service to terminate gracefully with the extended timeout that I am able to achieve when running the app locally from Visual Studio?
This is the only helpful link I could find in any Microsoft doc, but perhaps it doesn't apply to Azure App Services: https://learn.microsoft.com/en-us/aspnet/core/fundamentals/host/web-host?view=aspnetcore-8.0
Any help would be much appreciated!
There are a number of restrictions and limitations on azure app service. Adjusting the graceful shutdown period is not supported on multi-tenant app services (i.e. azure app service) to configure the shutdown period.
More details in this link:
https://learn.microsoft.com/en-us/answers/questions/1125435/extend-timeout-for-graceful-shutdown-in-azure-app
In terms of multi tenant app service: