Search code examples
springlogbackslf4jmdc

Logback & SLF4J Writing to multiple files using the MDCFilter


I have a spring project with these packages:

  • core
  • scheduledJobs
  • web
  • jms

Most of the logic is in the core package which is used by the other three packages. The other three packages do not use each others code.

I need to have 2 log files:

  • First log file should contain all log entries associated with scheduled jobs.
    • All log entries from the scheduledJobs package.
    • If I call a function from the core package from a function located in the scheduledJobs package then this entry should be only in this log file.
  • Second log file should contain every log entry except log entries that should go into the first log file.

The problem I can see the following information in the logback debug log:

[10/23/23 16:38:53:064 UTC] 0000008e SystemOut     O   16:38:53,064 |-WARN in ch.qos.logback.core.model.processor.ImplicitModelHandler - Ignoring unknown property [turboFilter] in [ch.qos.logback.core.rolling.RollingFileAppender]

I can see the MDC values in logs but filtering does not work, both log files have all the log entries.

My logback file:

<!-- LOCAL -->
<configuration>
    <property name="APP.NAME" scope="context" value="App"/>
    <property name="APP.LOG.PATH" scope="context" value="logs/APP"/>

    <appender name="FILE.APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${APP.LOG.PATH}/${APP.NAME}.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- daily rollover -->
            <fileNamePattern>${APP.LOG.PATH}/${APP.NAME}%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>10MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
            <maxHistory>10</maxHistory>
        </rollingPolicy>

        <turboFilter class="ch.qos.logback.classic.turbo.MDCFilter">
            <MDCKey>specialLogger</MDCKey>
            <Value>QUARTZ</Value>
            <OnMatch>DENY</OnMatch>
            <OnMismatch>ACCEPT</OnMismatch>
        </turboFilter>

        <encoder>
            <charset>UTF-8</charset>
            <pattern>%d{dd.MM.YYYY HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{correlationId}] [%X{specialLogger}] - %msg%n</pattern>
        </encoder>
    </appender>

    <appender name="FILE.APPENDER.QUARTZ" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${APP.LOG.PATH}/${APP.NAME}_QUARTZ.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- daily rollover -->
            <fileNamePattern>${APP.LOG.PATH}/${APP.NAME}_QUARTZ%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>10MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
            <maxHistory>10</maxHistory>
        </rollingPolicy>

        <turboFilter class="ch.qos.logback.classic.turbo.MDCFilter">
            <MDCKey>specialLogger</MDCKey>
            <Value>QUARTZ</Value>
            <OnMatch>ACCEPT</OnMatch>
            <OnMismatch>DENY</OnMismatch>
        </turboFilter>

        <encoder>
            <charset>UTF-8</charset>
            <pattern>%d{dd.MM.YYYY HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{correlationId}] [%X{specialLogger}] - %msg%n</pattern>
        </encoder>
    </appender>

    <root level="info">
        <appender-ref ref="FILE.APPENDER" />
        <appender-ref ref="FILE.APPENDER.QUARTZ" />
    </root>

</configuration>

Dependencies:

  • logback-classic: 1.3.5
  • logback-core: 1.3.5
  • org.slf4j: 2.0.6

EDIT: I also tried logging only to the "FILE.APPENDER.QUARTZ" which should only log entries based on the scheduledJobs filter. It contained all log entries anyway.


Solution

  • I found the solution. It isn't possible to use turboFilters for this functionality. Instead I have to use a custom standard filter.

    My logback file:

    <!-- LOCAL -->
    <configuration debug="true">
        <property name="APP.NAME" scope="context" value="PdfGen"/>
        <property name="APP.LOG.PATH" scope="context" value="logs/APP"/>
    
        <appender name="FILE.APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
            <file>${APP.LOG.PATH}/${APP.NAME}.log</file>
            <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
                <!-- daily rollover -->
                <fileNamePattern>${APP.LOG.PATH}/${APP.NAME}%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
                <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                    <maxFileSize>10MB</maxFileSize>
                </timeBasedFileNamingAndTriggeringPolicy>
                <maxHistory>10</maxHistory>
            </rollingPolicy>
    
            <filter class="<package_path>.MDCFilterRegular">
                <MDCKey>specialLogger</MDCKey>
                <Value>QUARTZ</Value>
                <OnMatch>DENY</OnMatch>
                <OnMismatch>ACCEPT</OnMismatch>
            </filter>
    
            <encoder>
                <charset>UTF-8</charset>
                <pattern>%d{dd.MM.YYYY HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{uniqueNumber}] [%X{correlationId}] - %msg%n</pattern>
            </encoder>
        </appender>
    
        <appender name="FILE.APPENDER.QUARTZ" class="ch.qos.logback.core.rolling.RollingFileAppender">
            <file>${APP.LOG.PATH}/${APP.NAME}_QUARTZ.log</file>
            <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
                <!-- daily rollover -->
                <fileNamePattern>${APP.LOG.PATH}/${APP.NAME}_QUARTZ%d{yyyy-MM-dd}.%i.zip</fileNamePattern>
                <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                    <maxFileSize>10MB</maxFileSize>
                </timeBasedFileNamingAndTriggeringPolicy>
                <maxHistory>10</maxHistory>
            </rollingPolicy>
    
            <filter class="<package_path>.MDCFilterRegular">
                <MDCKey>specialLogger</MDCKey>
                <Value>QUARTZ</Value>
                <OnMatch>ACCEPT</OnMatch>
                <OnMismatch>DENY</OnMismatch>
            </filter>
    
            <encoder>
                <charset>UTF-8</charset>
                <pattern>%d{dd.MM.YYYY HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{uniqueNumber}] [%X{correlationId}] - %msg%n</pattern>
            </encoder>
        </appender>
    
        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
            <!-- encoders are assigned the type
                 ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
            <encoder>
                <charset>UTF-8</charset>
                <pattern>%d{dd.MM.YYYY HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
            </encoder>
        </appender>
    
        <root>
            <appender-ref ref="STDOUT" />
            <appender-ref ref="FILE.APPENDER.QUARTZ" />
            <appender-ref ref="FILE.APPENDER" />
        </root>
    
    </configuration>
    

    The custom filter:

    import ch.qos.logback.classic.spi.ILoggingEvent;
    import ch.qos.logback.core.filter.AbstractMatcherFilter;
    import ch.qos.logback.core.spi.FilterReply;
    
    public class MDCFilterRegular extends AbstractMatcherFilter<ILoggingEvent> {
        String MDCKey;
        String Value;
    
        public MDCFilterRegular() {
        }
    
        public void start() {
            if (this.MDCKey != null && this.Value != null) {
                super.start();
            }
    
        }
    
        @Override
        public FilterReply decide(ILoggingEvent event) {
            if(!event.getMDCPropertyMap().containsKey(this.MDCKey)) {
                return this.onMismatch;
            }
    
            return event.getMDCPropertyMap().get(this.MDCKey).equals(Value) ? this.onMatch : this.onMismatch;
        }
    
        public String getMDCKey() {
            return MDCKey;
        }
    
        public void setMDCKey(String MDCKey) {
            this.MDCKey = MDCKey;
        }
    
        public String getValue() {
            return Value;
        }
    
        public void setValue(String value) {
            this.Value = value;
        }
    }