Search code examples
javadockeraws-ecsaws-elb

AWS ECS: Task Being Mapped to Wrong Target Group


I have a Spring project with 2 modules: Pets and Owners, each a microservice that gets its data from a single MySQL DB. When I push the two into a single Docker container with 2 tasks -- 1 for each microservice -- they both get up and running, and the tasks are mapped to different host ports (32768 & 32769). However, when I go to http://{elb-dns}/pet, I get a 404, but when I go to http://{elb-dns}/owner, it returns the expected data. I've learned the pet task gets mapped to the wrong Target Group, but idk why.

After creating the ECS cluster, this is what the following AWS CLI "describe" commands output:

aws ecs describe-clusters --cluster vets --region us-west-2
{
    "clusters": [
        {
            "status": "ACTIVE", 
            "clusterName": "vets", 
            "registeredContainerInstancesCount": 3, 
            "pendingTasksCount": 0, 
            "runningTasksCount": 4, 
            "activeServicesCount": 2, 
            "clusterArn": "arn:aws:ecs:us-west-2:224265390743:cluster/vets"
        }
    ], 
    "failures": []
}

aws elbv2 describe-target-groups --name vets0-tg --region us-west-2
{
    "TargetGroups": [
        {
            "HealthCheckPath": "/", 
            "HealthCheckIntervalSeconds": 60, 
            "VpcId": "vpc-0540fe4c35343c7c9", 
            "Protocol": "HTTP", 
            "HealthCheckTimeoutSeconds": 30, 
            "HealthCheckProtocol": "HTTP", 
            "LoadBalancerArns": [
                "arn:aws:elasticloadbalancing:us-west-2:224265390743:loadbalancer/app/vets-elb/5643eac9e4d51da0"
            ], 
            "UnhealthyThresholdCount": 3, 
            "HealthyThresholdCount": 5, 
            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets0-tg/8fec38deef74b8f1", 
            "Matcher": {
                "HttpCode": "200"
            }, 
            "HealthCheckPort": "traffic-port", 
            "Port": 8080, 
            "TargetGroupName": "vets0-tg"
        }
    ]
}

aws elbv2 describe-target-groups --name vets1-tg --region us-west-2
{
    "TargetGroups": [
        {
            "HealthCheckPath": "/", 
            "HealthCheckIntervalSeconds": 60, 
            "VpcId": "vpc-0540fe4c35343c7c9", 
            "Protocol": "HTTP", 
            "HealthCheckTimeoutSeconds": 30, 
            "HealthCheckProtocol": "HTTP", 
            "LoadBalancerArns": [
                "arn:aws:elasticloadbalancing:us-west-2:224265390743:loadbalancer/app/vets-elb/5643eac9e4d51da0"
            ], 
            "UnhealthyThresholdCount": 3, 
            "HealthyThresholdCount": 5, 
            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets1-tg/b4ca3cafcaebdc78", 
            "Matcher": {
                "HttpCode": "200"
            }, 
            "HealthCheckPort": "traffic-port", 
            "Port": 8080, 
            "TargetGroupName": "vets1-tg"
        }
    ]
}

aws ecs describe-task-definition --task-definition vets-rest-pet:25 --region us-west-2
{
    "taskDefinition": {
        "status": "ACTIVE", 
        "networkMode": "bridge", 
        "family": "vets-rest-pet", 
        "requiresAttributes": [
            {
                "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.ecr-auth"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.task-iam-role"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
            }
        ], 
        "volumes": [], 
        "taskRoleArn": "arn:aws:iam::224265390743:role/MicroECSTaskRole", 
        "taskDefinitionArn": "arn:aws:ecs:us-west-2:224265390743:task-definition/vets-rest-pet:25", 
        "containerDefinitions": [
            {
                "environment": [
                    {
                        "name": "SPRING_DATASOURCE_USERNAME", 
                        "value": "***"
                    }, 
                    {
                        "name": "SERVICE_ENDPOINT", 
                        "value": "***.us-west-2.elb.amazonaws.com"
                    }, 
                    {
                        "name": "SPRING_DATASOURCE_URL", 
                        "value": "***"
                    }, 
                    {
                        "name": "SPRING_DATASOURCE_PASSWORD", 
                        "value": "***"
                    }, 
                    {
                        "name": "SPRING_PROFILES_ACTIVE", 
                        "value": "mysql"
                    }
                ], 
                "name": "vets-rest-pet", 
                "mountPoints": [], 
                "image": "224265390743.dkr.ecr.us-west-2.amazonaws.com/vets-rest-pet:latest", 
                "dockerLabels": {
                    "string": "string"
                }, 
                "cpu": 1024, 
                "portMappings": [
                    {
                        "protocol": "tcp", 
                        "containerPort": 8080, 
                        "hostPort": 0
                    }
                ], 
                "logConfiguration": {
                    "logDriver": "awslogs", 
                    "options": {
                        "awslogs-region": "us-west-2", 
                        "awslogs-stream-prefix": "vets", 
                        "awslogs-group": "ECSLogGroup-vets"
                    }
                }, 
                "memory": 1024, 
                "essential": true, 
                "volumesFrom": []
            }
        ], 
        "revision": 25
    }
}

aws ecs describe-task-definition --task-definition vets-rest-owner:18 --region us-west-2
{
    "taskDefinition": {
        "status": "ACTIVE", 
        "networkMode": "bridge", 
        "family": "vets-rest-owner", 
        "requiresAttributes": [
            {
                "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.ecr-auth"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.task-iam-role"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
            }, 
            {
                "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
            }
        ], 
        "volumes": [], 
        "taskRoleArn": "arn:aws:iam::224265390743:role/MicroECSTaskRole", 
        "taskDefinitionArn": "arn:aws:ecs:us-west-2:224265390743:task-definition/vets-rest-owner:18", 
        "containerDefinitions": [
            {
                "environment": [
                    {
                        "name": "SPRING_DATASOURCE_USERNAME", 
                        "value": "***"
                    }, 
                    {
                        "name": "SERVICE_ENDPOINT", 
                        "value": "***.us-west-2.elb.amazonaws.com"
                    }, 
                    {
                        "name": "SPRING_DATASOURCE_URL", 
                        "value": "***"
                    }, 
                    {
                        "name": "SPRING_DATASOURCE_PASSWORD", 
                        "value": "***"
                    }, 
                    {
                        "name": "SPRING_PROFILES_ACTIVE", 
                        "value": "mysql"
                    }
                ], 
                "name": "vets-rest-owner", 
                "mountPoints": [], 
                "image": "224265390743.dkr.ecr.us-west-2.amazonaws.com/vets-rest-owner:latest", 
                "dockerLabels": {
                    "string": "string"
                }, 
                "cpu": 1024, 
                "portMappings": [
                    {
                        "protocol": "tcp", 
                        "containerPort": 8080, 
                        "hostPort": 0
                    }
                ], 
                "logConfiguration": {
                    "logDriver": "awslogs", 
                    "options": {
                        "awslogs-region": "us-west-2", 
                        "awslogs-stream-prefix": "vets", 
                        "awslogs-group": "ECSLogGroup-vets"
                    }
                }, 
                "memory": 1024, 
                "essential": true, 
                "volumesFrom": []
            }
        ], 
        "revision": 18
    }
}

aws elbv2 describe-load-balancers --region us-west-2
{
    "LoadBalancers": [
        {
            "VpcId": "vpc-0540fe4c35343c7c9", 
            "LoadBalancerArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:loadbalancer/app/vets-elb/5643eac9e4d51da0", 
            "State": {
                "Code": "active"
            }, 
            "DNSName": "***.us-west-2.elb.amazonaws.com", 
            "SecurityGroups": [
                "sg-09beda52a500a27fb"
            ], 
            "LoadBalancerName": "vets-elb", 
            "CreatedTime": "2018-03-27T02:55:14.510Z", 
            "Scheme": "internet-facing", 
            "Type": "application", 
            "CanonicalHostedZoneId": "Z1H1FL5HABSF5", 
            "AvailabilityZones": [
                {
                    "SubnetId": "subnet-04bf10c7538cae458", 
                    "ZoneName": "us-west-2c"
                }, 
                {
                    "SubnetId": "subnet-070b0cf7074abe6a4", 
                    "ZoneName": "us-west-2b"
                }, 
                {
                    "SubnetId": "subnet-0a88aaf68def9cd1e", 
                    "ZoneName": "us-west-2a"
                }
            ]
        }
    ]
}

aws elbv2 describe-listeners --load-balancer-arn arn:aws:elasticloadbalancing:us-west-2:224265390743:loadbalancer/app/vets-elb/5643eac9e4d51da0 --region us-west-2
{
    "Listeners": [
        {
            "Protocol": "HTTP", 
            "DefaultActions": [
                {
                    "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets-elb-tg/fff73d84613d20d9", 
                    "Type": "forward"
                }
            ], 
            "LoadBalancerArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:loadbalancer/app/vets-elb/5643eac9e4d51da0", 
            "Port": 80, 
            "ListenerArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:listener/app/vets-elb/5643eac9e4d51da0/0ff4e63088aede1c"
        }
    ]
}

aws elbv2 describe-rules --listener-arn arn:aws:elasticloadbalancing:us-west-2:224265390743:listener/app/vets-elb/5643eac9e4d51da0/0ff4e63088aede1c --region us-west-2
{
    "Rules": [
        {
            "Priority": "363", 
            "Conditions": [
                {
                    "Field": "path-pattern", 
                    "Values": [
                        "/*"
                    ]
                }
            ], 
            "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:listener-rule/app/vets-elb/5643eac9e4d51da0/0ff4e63088aede1c/d8fdc1195564af14", 
            "IsDefault": false, 
            "Actions": [
                {
                    "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets0-tg/8fec38deef74b8f1", 
                    "Type": "forward"
                }
            ]
        }, 
        {
            "Priority": "458", 
            "Conditions": [
                {
                    "Field": "path-pattern", 
                    "Values": [
                        "/*"
                    ]
                }
            ], 
            "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:listener-rule/app/vets-elb/5643eac9e4d51da0/0ff4e63088aede1c/40d6a6e2dd71eaa5", 
            "IsDefault": false, 
            "Actions": [
                {
                    "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets1-tg/b4ca3cafcaebdc78", 
                    "Type": "forward"
                }
            ]
        }, 
        {
            "Priority": "default", 
            "Conditions": [], 
            "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:listener-rule/app/vets-elb/5643eac9e4d51da0/0ff4e63088aede1c/24df0f9b37383df2", 
            "IsDefault": true, 
            "Actions": [
                {
                    "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets-elb-tg/fff73d84613d20d9", 
                    "Type": "forward"
                }
            ]
        }
    ]
}

I've confirmed that when deployed separately each microservice works in isolation. I'm using an Application Load Balancer, and each microservice has its own target group. I'm not sure what else to try or check. Has anyone experienced a similar problem? Where else should I look for clues?


Update 1

I changed the logging level to TRACE and learned that all /pet and /owner calls are being routed to the owner task exclusively. The following logs are from the owner task when I try http://{elb-dns}/pet/1:

9:00.590 DEBUG 7 - [nio-8080-exec-8] o.s.b.w.f.OrderedRequestContextFilter : Bound request context to thread: org.apache.catalina.connector.RequestFacade@446e699a
9:00.590 TRACE 7 - [nio-8080-exec-8] o.s.b.a.trace.WebRequestTraceFilter : Processing request GET /pet/1
9:00.590 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Bound request context to thread: org.apache.catalina.connector.RequestFacade@446e699a
9:00.590 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : DispatcherServlet with name 'dispatcherServlet' processing GET request for [/pet/1]
9:00.590 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.handler.SimpleUrlHandlerMapping@29ef6856] in DispatcherServlet with name 'dispatcherServlet'
9:00.590 TRACE 7 - [nio-8080-exec-8] o.s.w.s.handler.SimpleUrlHandlerMapping : No handler mapping found for [/pet/1]
9:00.590 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.boot.actuate.endpoint.mvc.EndpointHandlerMapping@2f162cc0] in DispatcherServlet with name 'dispatcherServlet'
9:00.590 DEBUG 7 - [nio-8080-exec-8] o.s.b.a.e.mvc.EndpointHandlerMapping : Looking up handler method for path /pet/1
9:00.591 DEBUG 7 - [nio-8080-exec-8] o.s.b.a.e.mvc.EndpointHandlerMapping : Did not find handler method for [/pet/1]
9:00.591 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping@b40bb6e] in DispatcherServlet with name 'dispatcherServlet'
9:00.591 DEBUG 7 - [nio-8080-exec-8] s.w.s.m.m.a.RequestMappingHandlerMapping : Looking up handler method for path /pet/1
9:00.591 DEBUG 7 - [nio-8080-exec-8] s.w.s.m.m.a.RequestMappingHandlerMapping : Did not find handler method for [/pet/1]
9:00.591 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.handler.BeanNameUrlHandlerMapping@34a75079] in DispatcherServlet with name 'dispatcherServlet'
9:00.591 TRACE 7 - [nio-8080-exec-8] o.s.w.s.h.BeanNameUrlHandlerMapping : No handler mapping found for [/pet/1]
9:00.591 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.data.rest.webmvc.support.DelegatingHandlerMapping@841e575] in DispatcherServlet with name 'dispatcherServlet'
9:00.591 DEBUG 7 - [nio-8080-exec-8] o.s.d.r.w.BasePathAwareHandlerMapping : Looking up handler method for path /pet/1
9:00.591 DEBUG 7 - [nio-8080-exec-8] o.s.d.r.w.BasePathAwareHandlerMapping : Did not find handler method for [/pet/1]
9:00.591 DEBUG 7 - [nio-8080-exec-8] o.s.d.r.w.RepositoryRestHandlerMapping : Looking up handler method for path /pet/1
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.d.r.w.RepositoryRestHandlerMapping : Found 1 matching mapping(s) for [/pet/1] : [{[/{repository}/{id}],methods=[GET],produces=[application/hal+json]}]
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.d.r.w.RepositoryRestHandlerMapping : Did not find handler method for [/pet/1]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.handler.SimpleUrlHandlerMapping@346a361] in DispatcherServlet with name 'dispatcherServlet'
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.w.s.handler.SimpleUrlHandlerMapping : Matching patterns for request [/pet/1] are [\/**]
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.w.s.handler.SimpleUrlHandlerMapping : URI Template variables for request [/pet/1] are {}
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.w.s.handler.SimpleUrlHandlerMapping : Mapping [/pet/1] to HandlerExecutionChain with handler [ResourceHttpRequestHandler [locations=[ServletContext resource [/], class path resource [META-INF/resources/], class path resource [resources/], class path resource [static/], class path resource [public/]], resolvers=[org.springframework.web.servlet.resource.PathResourceResolver@5471388b]]] and 1 interceptor
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler adapter [org.springframework.data.rest.webmvc.RepositoryRestHandlerAdapter@94f6bfb]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler adapter [org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter@2cac4385]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler adapter [org.springframework.web.servlet.mvc.HttpRequestHandlerAdapter@186978a6]
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Last-Modified value for [/pet/1] is: -1
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.r.ResourceHttpRequestHandler : Applying "invalid path" checks to path: pet/1
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Resolving resource for request path "pet/1"
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Checking location: ServletContext resource [/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : No match for location: ServletContext resource [/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Checking location: class path resource [META-INF/resources/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : No match for location: class path resource [META-INF/resources/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Checking location: class path resource [resources/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : No match for location: class path resource [resources/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Checking location: class path resource [static/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : No match for location: class path resource [static/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : Checking location: class path resource [public/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.resource.PathResourceResolver : No match for location: class path resource [public/]
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.w.s.r.ResourceHttpRequestHandler : No matching resource found - returning 404
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Null ModelAndView returned to DispatcherServlet with name 'dispatcherServlet': assuming HandlerAdapter completed request handling
9:00.592 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Cleared thread-bound request context: org.apache.catalina.connector.RequestFacade@446e699a
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Successfully completed request
9:00.592 TRACE 7 - [nio-8080-exec-8] ationConfigEmbeddedWebApplicationContext : Publishing event in org.springframework.boot.context.embedded.AnnotationConfigEmbeddedWebApplicationContext@77a567e1: ServletRequestHandledEvent: url=[/pet/1]; pet=[10.0.1.219]; method=[GET]; servlet=[dispatcherServlet]; session=[null]; user=[null]; time=[2ms]; status=[OK]
9:00.592 DEBUG 7 - [nio-8080-exec-8] o.s.b.w.f.OrderedRequestContextFilter : Cleared thread-bound request context: org.apache.catalina.connector.RequestFacade@446e699a
9:00.593 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Bound request context to thread: org.apache.catalina.core.ApplicationHttpRequest@6a03dc7
9:00.593 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : DispatcherServlet with name 'dispatcherServlet' processing GET request for [/error]
9:00.593 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.handler.SimpleUrlHandlerMapping@29ef6856] in DispatcherServlet with name 'dispatcherServlet'
9:00.593 TRACE 7 - [nio-8080-exec-8] o.s.w.s.handler.SimpleUrlHandlerMapping : No handler mapping found for [/error]
9:00.593 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.boot.actuate.endpoint.mvc.EndpointHandlerMapping@2f162cc0] in DispatcherServlet with name 'dispatcherServlet'
9:00.593 DEBUG 7 - [nio-8080-exec-8] o.s.b.a.e.mvc.EndpointHandlerMapping : Looking up handler method for path /error
9:00.593 DEBUG 7 - [nio-8080-exec-8] o.s.b.a.e.mvc.EndpointHandlerMapping : Did not find handler method for [/error]
9:00.593 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler map [org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping@b40bb6e] in DispatcherServlet with name 'dispatcherServlet'
9:00.593 DEBUG 7 - [nio-8080-exec-8] s.w.s.m.m.a.RequestMappingHandlerMapping : Looking up handler method for path /error
9:00.593 TRACE 7 - [nio-8080-exec-8] s.w.s.m.m.a.RequestMappingHandlerMapping : Found 2 matching mapping(s) for [/error] : [{[/error],produces=[text/html]}, {[/error]}]
9:00.594 DEBUG 7 - [nio-8080-exec-8] s.w.s.m.m.a.RequestMappingHandlerMapping : Returning handler method [public org.springframework.web.servlet.ModelAndView org.springframework.boot.autoconfigure.web.BasicErrorController.errorHtml(javax.servlet.http.HttpServletRequest,javax.servlet.http.HttpServletResponse)]
9:00.594 DEBUG 7 - [nio-8080-exec-8] o.s.b.f.s.DefaultListableBeanFactory : Returning cached instance of singleton bean 'basicErrorController'
9:00.594 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler adapter [org.springframework.data.rest.webmvc.RepositoryRestHandlerAdapter@94f6bfb]
9:00.594 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Testing handler adapter [org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter@2cac4385]
9:00.594 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Last-Modified value for [/error] is: -1
9:00.594 DEBUG 7 - [nio-8080-exec-8] o.j.s.OpenEntityManagerInViewInterceptor : Opening JPA EntityManager in OpenEntityManagerInViewInterceptor
9:00.594 TRACE 7 - [nio-8080-exec-8] .s.t.s.TransactionSynchronizationManager : Bound value [org.springframework.orm.jpa.EntityManagerHolder@556e96bc] for key [org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean@47c4ecdc] to thread [http-nio-8080-exec-8]
9:00.594 TRACE 7 - [nio-8080-exec-8] .w.s.m.m.a.ServletInvocableHandlerMethod : Invoking 'org.springframework.boot.autoconfigure.web.BasicErrorController.errorHtml' with arguments [org.apache.catalina.core.ApplicationHttpRequest@6a03dc7, org.apache.catalina.connector.ResponseFacade@1a8818e]
9:00.595 TRACE 7 - [nio-8080-exec-8] .w.s.m.m.a.ServletInvocableHandlerMethod : Method [org.springframework.boot.autoconfigure.web.BasicErrorController.errorHtml] returned [ModelAndView: reference to view with name 'error'; model is {timestamp=Sat Mar 24 9:00 GMT 2018, status=404, error=Not Found, message=No message available, path=/pet/1}]
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.w.s.v.ContentNegotiatingViewResolver : Requested media types are [text/html, text/html;q=0.8] based on Accept header types and producible media types [text/html])
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.b.f.s.DefaultListableBeanFactory : Returning cached instance of singleton bean 'error'
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.w.s.v.ContentNegotiatingViewResolver : Returning [org.springframework.boot.autoconfigure.web.ErrorMvcAutoConfiguration$SpelView@7a8fa663] based on requested media type 'text/html'
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Rendering view [org.springframework.boot.autoconfigure.web.ErrorMvcAutoConfiguration$SpelView@7a8fa663] in DispatcherServlet with name 'dispatcherServlet'
9:00.595 TRACE 7 - [nio-8080-exec-8] o.s.util.PropertyPlaceholderHelper : Resolved placeholder 'timestamp'
9:00.595 TRACE 7 - [nio-8080-exec-8] o.s.util.PropertyPlaceholderHelper : Resolved placeholder 'error'
9:00.595 TRACE 7 - [nio-8080-exec-8] o.s.util.PropertyPlaceholderHelper : Resolved placeholder 'status'
9:00.595 TRACE 7 - [nio-8080-exec-8] o.s.util.PropertyPlaceholderHelper : Resolved placeholder 'message'
9:00.595 TRACE 7 - [nio-8080-exec-8] .s.t.s.TransactionSynchronizationManager : Removed value [org.springframework.orm.jpa.EntityManagerHolder@556e96bc] for key [org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean@47c4ecdc] from thread [http-nio-8080-exec-8]
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.j.s.OpenEntityManagerInViewInterceptor : Closing JPA EntityManager in OpenEntityManagerInViewInterceptor
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.orm.jpa.EntityManagerFactoryUtils : Closing JPA EntityManager
9:00.595 TRACE 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Cleared thread-bound request context: org.apache.catalina.core.ApplicationHttpRequest@6a03dc7
9:00.595 DEBUG 7 - [nio-8080-exec-8] o.s.web.servlet.DispatcherServlet : Successfully completed request
9:00.595 TRACE 7 - [nio-8080-exec-8] ationConfigEmbeddedWebApplicationContext : Publishing event in org.springframework.boot.context.embedded.AnnotationConfigEmbeddedWebApplicationContext@77a567e1: ServletRequestHandledEvent: url=[/error]; pet=[10.0.1.219]; method=[GET]; servlet=[dispatcherServlet]; session=[null]; user=[null]; time=[2ms]; status=[OK]

That same call produces no logs in the pet task, indicating that it's not getting routed to the correct task in the container. So I'm still stuck on this problem.


Update 2

The ELB logs show that, for both /pet and /owner, all calls are being routed to the same Target Group. However, /pet should be routed to one TG, and /owner should be routed to the other. To be clear, I have 3 TG's: one on container port 8080 for /pet, one on container port 8080 for /owner, and one on container port 80 that accepts HTTP requests. All three TG's are in the same VPC.

http 03:48:22.439446Z app/vets-elb/ab3d7952d0ea2843 76.102.41.144:61442 10.0.12.74:32768 0.002 0.020 0.000 200 200 263 1259 "GET http://vets-elb-1925600148.us-west-2.elb.amazonaws.com:80/owner HTTP/1.1" "PostmanRuntime/6.4.1" - - arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets0-tg/16a5158616c45834 "Root=1-5ab71c06-4798a50a19129492453fd34c" "-" "-" 52
http 03:48:29.581797Z app/vets-elb/ab3d7952d0ea2843 76.102.41.144:61442 10.0.11.42:32768 0.000 0.006 0.000 404 404 263 319 "GET http://vets-elb-1925600148.us-west-2.elb.amazonaws.com:80/pet HTTP/1.1" "PostmanRuntime/6.4.1" - - arn:aws:elasticloadbalancing:us-west-2:224265390743:targetgroup/vets0-tg/16a5158616c45834 "Root=1-5ab71c0d-cff47d0a6bb893c0bc1b9b88" "-" "-" 52

Solution

  • The "rules" section of your ALB setup looks like the problem to me. They both have the pattern set to /* ... which means you don't tell it anywhere that /pets/* goes to one target and /owners/* to another.