Search code examples
dockerjenkinsgitlabrestart

Jenkins Container restarting after creating Gitlab container


This is my docker-compose file:

version: '3'
services:
  remote_host:
    container_name: remote-host
    image: remote-host
    build:
      context: centos7
    networks:
      - net
  jenkins:
    container_name: jenkins
    image: jenkins-blueocean
    ports:
      - "8081:8080"
    build:
      context: .
    volumes:
      - "$PWD/jenkins_home:/var/jenkins_home"
    restart: on-failure
    networks:
      - net
  git:
    container_name: git-server
    image: 'gitlab/gitlab-ee:latest'
    hostname: 'gitlab.example.com'
    ports:
      - '8091:80'
    volumes:
      - '$GITLAB_HOME/config:/etc/gitlab'
      - '$GITLAB_HOME/logs:/var/log/gitlab'
      - '$GITLAB_HOME/data:/var/opt/gitlab'
    networks:
      - net

networks:
  net:

All containers get up and running successfully but as soon as I am logging into Jenkins UI, the corresponding container crashes.

If I stop Gitlab container, I am not facing this issue.

I am using these command to run the containers:

export GITLAB_HOME=$HOME/gitlab
docker-compose up -d

Only error I can see in the Jenkins logs is as below

    at java.base/java.lang.Thread.run(Thread.java:829)
2023-03-26 14:23:36 Caused by: java.lang.AssertionError: class hudson.security.csrf.DefaultCrumbIssuer is missing its descriptor
2023-03-26 14:23:36     at jenkins.model.Jenkins.getDescriptorOrDie(Jenkins.java:1634)
2023-03-26 14:23:36     at hudson.security.csrf.CrumbIssuer.getDescriptor(CrumbIssuer.java:141)
2023-03-26 14:23:36     at hudson.Functions.getCrumbRequestField(Functions.java:1997)
2023-03-26 14:23:36     at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2023-03-26 14:23:36     at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2023-03-26 14:23:36     at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2023-03-26 14:23:36     at java.base/java.lang.reflect.Method.invoke(Method.java:566)
2023-03-26 14:23:36     at org.apache.commons.jexl.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:258)
2023-03-26 14:23:36     at org.apache.commons.jexl.parser.ASTMethod.execute(ASTMethod.java:104)
2023-03-26 14:23:36     at org.apache.commons.jexl.parser.ASTReference.execute(ASTReference.java:83)
2023-03-26 14:23:36     at org.apache.commons.jexl.parser.ASTReference.value(ASTReference.java:57)
2023-03-26 14:23:36     at org.apache.commons.jexl.parser.ASTReferenceExpression.value(ASTReferenceExpression.java:51)
2023-03-26 14:23:36     at org.apache.commons.jexl.ExpressionImpl.evaluate(ExpressionImpl.java:80)
2023-03-26 14:23:36     at hudson.ExpressionFactory2$JexlExpression.evaluate(ExpressionFactory2.java:76)
2023-03-26 14:23:36     at org.apache.commons.jelly.expression.ExpressionSupport.evaluateRecurse(ExpressionSupport.java:61)
2023-03-26 14:23:36     at org.apache.commons.jelly.expression.ExpressionSupport.evaluateAsString(ExpressionSupport.java:46)
2023-03-26 14:23:36     at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.buildAttributes(ReallyStaticTagLibrary.java:113)
2023-03-26 14:23:36     at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:97)
2023-03-26 14:23:36     at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
2023-03-26 14:23:36     at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:101)
2023-03-26 14:23:36     at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
2023-03-26 14:23:36     at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:101)
2023-03-26 14:23:36     at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
2023-03-26 14:23:36     at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:265)
2023-03-26 14:23:36     ... 58 more
2023-03-26 14:23:36 Caused: javax.servlet.ServletException
2023-03-26 14:23:36     at org.kohsuke.stapler.Facet$1.dispatch(Facet.java:249)
2023-03-26 14:23:36     at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:762)
2023-03-26 14:23:36     at org.kohsuke.stapler.Stapler.invoke(Stapler.java:894)
2023-03-26 14:23:36     at org.kohsuke.stapler.Stapler.invoke(Stapler.java:690)
2023-03-26 14:23:36     at hudson.init.impl.InstallUncaughtExceptionHandler.handleException(InstallUncaughtExceptionHandler.java:59)
2023-03-26 14:23:36     at hudson.init.impl.InstallUncaughtExceptionHandler.lambda$init$0(InstallUncaughtExceptionHandler.java:33)
2023-03-26 14:23:36     at org.kohsuke.stapler.compression.CompressionFilter.reportException(CompressionFilter.java:72)
2023-03-26 14:23:36     at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:56)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
2023-03-26 14:23:36     at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:86)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
2023-03-26 14:23:36     at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
2023-03-26 14:23:36     at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:38)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
2023-03-26 14:23:36     at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
2023-03-26 14:23:36     at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:549)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
2023-03-26 14:23:36     at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1571)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1383)
2023-03-26 14:23:36     at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)

Solution

  • It got killed due to out of memory. I inspected the killed container and found exit code 137.

    "State": {
                "Status": "exited",
                "Running": false,
                "Paused": false,
                "Restarting": false,
                "OOMKilled": true,
                "Dead": false,
                "Pid": 0,
                "ExitCode": 137,
                "Error": "",
                "StartedAt": "2023-03-29T10:26:03.135664452Z",
                "FinishedAt": "2023-03-29T10:28:49.133673682Z"
            }
    

    It seems GitLab container consumes more memory and increasing the burden on docker resources.

    So I increased memory from 4Gb to 8Gb in docker desktop.