The web application is a REST API and a SPA. It's maintained and currently on .NET 6.0 and have been working steadily for years.
The requests are CORS and the server is properly configured for this.
Suddenly we have several outbursts per day of POST and PUT requests consistently failing with 500 server error. And they are failing quickly, only 30 ms. This goes on for 5-15 minutes and then returns to normal again.
All GET requests still working perfectly fine in between, which is strange.
Even stranger these failing requests are not logged, like they never reach the web server. Checked both web server logs (IIS) and ASP.NET Core application exceptions and traces.
The response header X-Powered-By: ASP.NET is also missing from these failing requests. Which would be present for normal 500 server errors.
Also suggesting the requests never reach the server.
The App Service Plan is only using 30% of it's resources currently.
Same behaviour is confirmed across Chrome, Edge and Safari. But you can have the issue in Chrome, while a session in Edge is working flawlessly on the same PC.
If we close the browser and re-open, the issues are gone.
It all started this month: May 2024.
Also worth mentioning that we have a DNS load balancer, Azure Traffic Manager.
This only operates on DNS queries and returns the closest of the two instances of the REST API services.
Traffic Manager does therefore not log any requests.
Update confirmed that error also occurs without Traffic Manager.
Update 2 We have tested deploying on a brand new App Service. And tried upgrading to .NET8.0. To no avail
That leaves us with browser's network log the only place to inspect. We have exported numbers of HAR files and looked for differences between failing requests and working requests, and found none.
Has anyone experienced similar behaviour of any kind?
Configuration of ASP.NET Core in Startup.cs:
public class Startup
{
readonly string _corsPolicyName = "corsOrigins";
public IConfiguration Configuration { get; }
public IWebHostEnvironment Environment { get; }
private static IConfiguration config;
public static IConfiguration GetConfiguration()
{
return config;
}
public Startup(IConfiguration configuration, IWebHostEnvironment environment)
{
Configuration = configuration;
config = configuration;
Environment = environment;
}
public void ConfigureServices(IServiceCollection services)
{
if (Environment.IsDevelopment())
IdentityModelEventSource.ShowPII = true;
// logging
services.AddApplicationInsightsTelemetry();
services.AddLogging(logging =>
{
logging.AddSimpleConsole();
logging.AddAzureWebAppDiagnostics();
});
services.Configure<AzureFileLoggerOptions>(options =>
{
options.FileName = "filelog-";
options.FileSizeLimit = 50 * 1024;
options.RetainedFileCountLimit = 5;
});
services.Configure<AzureBlobLoggerOptions>(options =>
{
options.BlobName = "Backend.txt";
});
// so our claims will not be translated
JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear();
// configure languages
services.Configure<RequestLocalizationOptions>(options =>
{
var supportedCultures = new[]
{
new CultureInfo("en"),
new CultureInfo("no")
};
options.DefaultRequestCulture = new RequestCulture("en");
options.SupportedCultures = supportedCultures;
options.SupportedUICultures = supportedCultures;
});
// add Identity
services.AddIdentity<ApplicationUser, ApplicationRole>()
.AddEntityFrameworkStores<AuthContext>()
.AddRoleManager<RoleManager<ApplicationRole>>()
.AddDefaultTokenProviders();
services.AddUserAndPasswordPolicies();
services.AddCors(options =>
{
{
options.AddPolicy(_corsPolicyName, builder =>
builder.SetIsOriginAllowedToAllowWildcardSubdomains()
.WithOrigins("https://*.our.domain")
.AllowAnyMethod()
.AllowAnyHeader()
.AllowCredentials());
}
});
services.AddAuthentication(options =>
{
options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
}).AddJwtBearer(o =>
{
o.MapInboundClaims = false;
o.Authority = Configuration.GetValue<string>("authServerUrl");
o.Audience = "backend";
o.RequireHttpsMetadata = true;
o.SaveToken = true;
o.TokenValidationParameters = new TokenValidationParameters
{
NameClaimType = "name",
RoleClaimType = "role",
ValidateIssuer = true,
ValidateAudience = true
};
});
services.AddControllersWithViews(options =>
{
options.Filters.Add<WebCustomExceptionFilter>();
})
.AddNewtonsoftJson(options =>
{
options.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
options.SerializerSettings.DateTimeZoneHandling = DateTimeZoneHandling.Utc;
});
services.AddSignalRManager();
services.AddRazorPages()
.AddRazorRuntimeCompilation();
services.AddSwaggerGenNewtonsoftSupport();
services.AddSwaggerGen(options =>
{
options.SwaggerDoc("v1", new OpenApiInfo { Title = "server debug api spec", Version = "v1" });
options.SchemaFilter<EnumSchemaFilter>();
});
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
StaticEnvironment.IsDebug = env.IsDevelopment();
StaticEnvironment.ContentRootPath = env.ContentRootPath;
app.UseCors(_corsPolicyName);
if (env.IsDevelopment())
{
app.UseStaticFiles();
app.UseDeveloperExceptionPage();
app.UseSwagger();
app.UseSwaggerUI(c => {
c.DocumentTitle = "Backend Swagger docs";
var sidebar = Path.Combine(env.ContentRootPath, "wwwroot/sidebar.html");
c.HeadContent = File.ReadAllText(sidebar);
c.InjectStylesheet("/colors.css");
c.InjectStylesheet("/style.css");
c.SwaggerEndpoint("/swagger/v1/swagger.json", "server");
});
}
else
{
app.UseHttpsRedirection();
app.UseHsts();
}
// Localization
app.UseRequestLocalization();
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
// Enable static files for use in MVC views.
app.UseStaticFiles();
app.UseStaticFiles(new StaticFileOptions()
{
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), @"Style")),
RequestPath = new PathString("/Style")
});
}
}
Typical controller signature:
[Authorize]
[ApiController]
[Route("api/[controller]")]
public class ImprovementController : ControllerBase
The Web client fetch code
const fullUrl = `${baseUrl}/${url}`
const bearer = getBearer()
const connectionId = getConnectionId()
const req = {
method: "POST",
headers: {
"Authorization": bearer,
"Content-Type": "application/json"
},
body: JSON.stringify(data)
}
const res = await fetch(fullUrl, req)
There are several tickets raised at Microsoft. Please see https://learn.microsoft.com/en-us/answers/questions/1687258/our-azure-app-service-application-started-to-exper
We have the same issue and a lot of others have. Post requests are failing randomly with 500 error and gets are working fine. After a while it works again. Apparently it can be fixed in the meantime by setting the setting of HTTP/2 back to HTTP/1.1
I hope to confirm this next monday when all our customers are back to work