I'm testing combination of Serilog.Sinks.Elasticsearch library and ELK stack (Elasticsearch & Kibana) for collecting logs from my ASP.NET Core 2.2 application. Web application and ELK services are containerized via Docker.
When spinning all containers via docker-compose everything is fine except from some reason Elasticsearch Sink is not pushing logs generated in HomeController.cs to Elasticsearch. This can be checked on http://localhost:9200/_cat/indices?v where no new index with logstash-* is generated.
What's interesting is that when running combination of WebApp via IIS (not in container) and ELK stack in container, logs are generated and sent to Elasticsearch. I'm assuming that something is wrong with port or network configuration of WebApp in docker-compose.yml file.
I've also tried to switch localhost with elasticsearch (actual name of elasticsearch container) in appsettings.Development.json but no effect.
Could anyone please assist me with this problem? Thank you.
Docker-compose.yml file:
version: '3.4'
services:
elastic.serilog.web:
image: ${DOCKER_REGISTRY-}elasticserilogweb
build:
context: .
dockerfile: Elastic.Serilog.Web/Dockerfile
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
container_name: elasticsearch
ports:
- "9200:9200"
volumes:
- elasticsearch-data:/usr/share/elasticsearch/data
environment:
- discovery.type=single-node
- xpack.security.enabled=false
- xpack.monitoring.enabled=true
- xpack.watcher.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
networks:
- docker-network
kibana:
image: docker.elastic.co/kibana/kibana:7.2.0
container_name: kibana
ports:
- "5601:5601"
depends_on:
- elasticsearch
environment:
- ELASTICSEARCH_HOSTS="http://elasticsearch:9200"
- XPACK_MONITORING_ENABLED=true
networks:
- docker-network
networks:
docker-network:
driver: bridge
volumes:
elasticsearch-data:
Docker-compose.override.yml file:
version: '3.4'
services:
elastic.serilog.web:
environment:
- ASPNETCORE_ENVIRONMENT=Development
ports:
- "80:7000"
Dockerfile:
FROM microsoft/dotnet:2.1-aspnetcore-runtime AS base
WORKDIR /app
EXPOSE 80
FROM microsoft/dotnet:2.1-sdk AS build
WORKDIR /src
COPY Elastic.Serilog.Web/Elastic.Serilog.Web.csproj Elastic.Serilog.Web/
RUN dotnet restore Elastic.Serilog.Web/Elastic.Serilog.Web.csproj
COPY . .
WORKDIR /src/Elastic.Serilog.Web
RUN dotnet build Elastic.Serilog.Web.csproj -c Release -o /app
FROM build AS publish
RUN dotnet publish Elastic.Serilog.Web.csproj -c Release -o /app
FROM base AS final
WORKDIR /app
COPY --from=publish /app .
ENTRYPOINT ["dotnet", "Elastic.Serilog.Web.dll"]
Appsettings.Development.json:
{
"Logging": {
"LogLevel": {
"Default": "Error",
"System": "Error",
"Microsoft": "Warning"
}
},
"AllowedHosts": "*",
"ElasticConfiguration": {
"Uri": "http://localhost:9200/"
}
}
Startup.cs:
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Exceptions;
using Serilog.Sinks.Elasticsearch;
using System;
namespace Elastic.Serilog.Web
{
public class Startup
{
public IConfiguration Configuration { get; }
public Startup(IConfiguration configuration, IHostingEnvironment hostingEnvironment)
{
var builder = new ConfigurationBuilder()
.SetBasePath(hostingEnvironment.ContentRootPath)
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile($"appsettings.{hostingEnvironment.EnvironmentName}.json", reloadOnChange: true, optional: true)
.AddEnvironmentVariables();
Configuration = builder.Build();
var elasticUri = Configuration["ElasticConfiguration:Uri"];
Log.Logger = new LoggerConfiguration()
.Enrich.FromLogContext()
.Enrich.WithExceptionDetails()
.Enrich.WithMachineName()
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(elasticUri))
{
AutoRegisterTemplate = true,
})
.CreateLogger();
}
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
}
public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
{
loggerFactory.AddSerilog();
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseExceptionHandler("/Home/Error");
app.UseHsts();
}
// app.UseHttpsRedirection();
app.UseStaticFiles();
// app.UseCookiePolicy();
app.UseMvc(routes =>
{
routes.MapRoute(
name: "default",
template: "{controller=Home}/{action=Index}/{id?}");
});
}
}
}
HomeController.cs
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Elastic.Serilog.Web.Models;
using Microsoft.Extensions.Logging;
using Serilog;
namespace Elastic.Serilog.Web.Controllers
{
public class HomeController : Controller
{
ILogger<HomeController> _logger;
public HomeController(ILogger<HomeController> logger)
{
_logger = logger;
}
public IActionResult Index()
{
_logger.LogInformation($"oh hai there! : {DateTime.UtcNow}");
Log.Error("Test Serilog!");
try
{
throw new Exception("oops. i haz cause error in UR codez.");
}
catch (Exception ex)
{
_logger.LogCritical("ur app haz critical error", ex);
_logger.LogError(ex, "ur code iz buggy.");
Log.Information("Test Serilog!");
}
return View();
}
public IActionResult About()
{
ViewData["Message"] = "Your application description page.";
return View();
}
public IActionResult Contact()
{
ViewData["Message"] = "Your contact page.";
return View();
}
public IActionResult Privacy()
{
return View();
}
[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
public IActionResult Error()
{
return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier });
}
}
}
Solved by:
1.) Make sure all services defined in docker-compose.yml are in the same network. In our case we must put all services in custom "docker-network". (If we skip defining custom network Docker will use default one, which is also in bridge mode - so theoretically all containers would still use the same network).
2.) Since we are running services in containers "localhost" in no more relevant for addressing port 9200 in appsettings.Development.json. We need to change localhost into actual name of Elastic Search container - in our case "elasticsearch":
"ElasticConfiguration": {
"Uri": "http://elasticsearch:9200/"
}
After following this two steps serilog-sinks-elasticsearch will push all logs to Elastic Search and further on default logstash-* index will be visible in Kibana.