Search code examples
aws-cloudformationaws-amplify

AWS Amplify won't deploy because of CustomAuthTriggerResource error in CloudFormation

When I created a pre-signup hook for Amplify, the following CloudFormation template was generated and it worked:


{
  "Description": "Custom Resource stack for Auth Trigger created using Amplify CLI",
  "AWSTemplateFormatVersion": "2010-09-09",
  "Parameters": {
    "env": {
      "Type": "String"
    },
    "userpoolId": {
      "Type": "String"
    },
    "userpoolArn": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupName": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupArn": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupLambdaExecutionRole": {
      "Type": "String"
    }
  },
  "Conditions": {
    "ShouldNotCreateEnvResources": {
      "Fn::Equals": [
        {
          "Ref": "env"
        },
        "NONE"
      ]
    }
  },
  "Resources": {
    "UserPoolPreSignUpLambdaInvokePermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "functionmyappwebappPreSignupName"
        },
        "Principal": "cognito-idp.amazonaws.com",
        "SourceArn": {
          "Ref": "userpoolArn"
        }
      }
    },
    "authTriggerFnServiceRole08093B67": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          {
            "Fn::Join": [
              "",
              [
                "arn:",
                {
                  "Ref": "AWS::Partition"
                },
                ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
              ]
            ]
          }
        ]
      }
    },
    "authTriggerFnServiceRoleDefaultPolicyEC9285A8": {
      "Type": "AWS::IAM::Policy",
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "cognito-idp:DescribeUserPool",
                "cognito-idp:UpdateUserPool"
              ],
              "Effect": "Allow",
              "Resource": {
                "Ref": "userpoolArn"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "Roles": [
          {
            "Ref": "authTriggerFnServiceRole08093B67"
          }
        ]
      }
    },
    "authTriggerFn7FCFA449": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "ZipFile": "const response = require('cfn-response');\nconst aws = require('aws-sdk');\n\nexports.handler = async function (event, context) {\n  const physicalResourceId =\n    event.RequestType === 'Update' ? event.PhysicalResourceId : `${event.LogicalResourceId}-${event.ResourceProperties.userpoolId}`;\n\n  try {\n    const userPoolId = event.ResourceProperties.userpoolId;\n    const { lambdaConfig } = event.ResourceProperties;\n    const config = {};\n    const cognitoClient = new aws.CognitoIdentityServiceProvider();\n    const userPoolConfig = await cognitoClient.describeUserPool({ UserPoolId: userPoolId }).promise();\n    const userPoolParams = userPoolConfig.UserPool;\n    // update userPool params\n\n    const updateUserPoolConfig = {\n      UserPoolId: userPoolParams.Id,\n      Policies: userPoolParams.Policies,\n      SmsVerificationMessage: userPoolParams.SmsVerificationMessage,\n      AccountRecoverySetting: userPoolParams.AccountRecoverySetting,\n      AdminCreateUserConfig: userPoolParams.AdminCreateUserConfig,\n      AutoVerifiedAttributes: userPoolParams.AutoVerifiedAttributes,\n      EmailConfiguration: userPoolParams.EmailConfiguration,\n      EmailVerificationMessage: userPoolParams.EmailVerificationMessage,\n      EmailVerificationSubject: userPoolParams.EmailVerificationSubject,\n      VerificationMessageTemplate: userPoolParams.VerificationMessageTemplate,\n      SmsAuthenticationMessage: userPoolParams.SmsAuthenticationMessage,\n      MfaConfiguration: userPoolParams.MfaConfiguration,\n      DeviceConfiguration: userPoolParams.DeviceConfiguration,\n      SmsConfiguration: userPoolParams.SmsConfiguration,\n      UserPoolTags: userPoolParams.UserPoolTags,\n      UserPoolAddOns: userPoolParams.UserPoolAddOns,\n    };\n\n    // removing undefined keys\n    Object.keys(updateUserPoolConfig).forEach((key) => updateUserPoolConfig[key] === undefined && delete updateUserPoolConfig[key]);\n\n    /* removing UnusedAccountValidityDays as deprecated\n    InvalidParameterException: Please use TemporaryPasswordValidityDays in PasswordPolicy instead of UnusedAccountValidityDays\n    */\n    if (updateUserPoolConfig.AdminCreateUserConfig && updateUserPoolConfig.AdminCreateUserConfig.UnusedAccountValidityDays) {\n      delete updateUserPoolConfig.AdminCreateUserConfig.UnusedAccountValidityDays;\n    }\n    lambdaConfig.forEach((lambda) => (config[`${lambda.triggerType}`] = lambda.lambdaFunctionArn));\n    if (event.RequestType === 'Delete') {\n      try {\n        updateUserPoolConfig.LambdaConfig = {};\n        console.log(`${event.RequestType}:`, JSON.stringify(updateUserPoolConfig));\n        const result = await cognitoClient.updateUserPool(updateUserPoolConfig).promise();\n        console.log(`delete response data ${JSON.stringify(result)}`);\n        await response.send(event, context, response.SUCCESS, {}, physicalResourceId);\n      } catch (err) {\n        console.log(err.stack);\n        await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n      }\n    }\n    if (event.RequestType === 'Update' || event.RequestType === 'Create') {\n      updateUserPoolConfig.LambdaConfig = config;\n      try {\n        const result = await cognitoClient.updateUserPool(updateUserPoolConfig).promise();\n        console.log(`createOrUpdate response data ${JSON.stringify(result)}`);\n        await response.send(event, context, response.SUCCESS, {}, physicalResourceId);\n      } catch (err) {\n        console.log(err.stack);\n        await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n      }\n    }\n  } catch (err) {\n    console.log(err.stack);\n    await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n  }\n};\n"
        },
        "Role": {
          "Fn::GetAtt": [
            "authTriggerFnServiceRole08093B67",
            "Arn"
          ]
        },
        "Handler": "index.handler",
        "Runtime": "nodejs14.x"
      },
      "DependsOn": [
        "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "authTriggerFnServiceRole08093B67"
      ]
    },
    "CustomAuthTriggerResource": {
      "Type": "Custom::CustomAuthTriggerResourceOutputs",
      "Properties": {
        "ServiceToken": {
          "Fn::GetAtt": [
            "authTriggerFn7FCFA449",
            "Arn"
          ]
        },
        "userpoolId": {
          "Ref": "userpoolId"
        },
        "lambdaConfig": [
          {
            "triggerType": "PreSignUp",
            "lambdaFunctionName": "myappwebappPreSignup",
            "lambdaFunctionArn": {
              "Ref": "functionmyappwebappPreSignupArn"
            }
          }
        ],
        "nonce": "cd55fb39-4ef8-416e-bc0b-178353ecc845"
      },
      "DependsOn": [
        "authTriggerFn7FCFA449",
        "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "authTriggerFnServiceRole08093B67"
      ],
      "UpdateReplacePolicy": "Delete",
      "DeletionPolicy": "Delete"
    }
  }
}

After some changes, I think I accidentally deleted authTriggerFn7FCFA449 and the whole deployment stopped working. CloudFormation started complaining like this:

Function not found: arn:aws:lambda:eu-central-1:510148651032:function:amplify-amplify65170398e4384-authTriggerFn7FCFA449-wBSraCi16QuH
(Service: AWSLambda; Status Code: 404; Error Code: ResourceNotFoundException; Request ID: 7195cef0-164c-4095-9de3-8a8309086a99; Proxy: null)

After detecting the drift I managed to recreate the Lambda, with all the roles, tags, code, cfn-response properly set up. Now, there's no drift. Additionally, the previous error dissapears.

However, CloudFormation waits for an hour and then is stuck at UPDATE_FAILED on CustomAuthTriggerResource with the following problem:

CloudFormation did not receive a response from your Custom Resource.
Please check your logs for requestId [44595a05-67c1-4594-b16b-096ce6506d96].
If you are using the Python cfn-response module, you may need to update your Lambda function code so that CloudFormation can attach the updated version.

All other resources (authTriggerFn7FCFA449, authTriggerFnServiceRole08093B67, authTriggerFnServiceRoleDefaultPolicyEC9285A8, UserPoolPreSignUpLambdaInvokePermission) are at CREATE_COMPLETE and UPDATE_COMPLETE.

Looking at the amplify-amplify65170398e4384-authTriggerFn7FCFA449-wBSraCi16QuH logs, there is no such request. How do I proceed from here?

No git pushes, amplify pushes or stack updates fix the situation.

Edit: I think this is related: GitHub: Make the CustomAuthTriggerResource timeout configurable #9837

Solution

  • Seems like this is a known problem. Solved using GitHub Issue: Make the CustomAuthTriggerResource timeout configurable and the AWS documentation on stuck resources

    Workaround

    1. Add one line showing the CloudFormation event at the beginning of the current Lambda function ...authTriggerFn7FCFA449 via Lambda console. For example, console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
    2. Click the "continue update rollback" button on CloudFormation console as the AWS documentation shows
    3. Watch the CloudWatch log stream of the above Lambda function, note the S3 pre-signed URL, PhysicalResourceId, StackId, LogicalResourceId. An output will look like
    {
    "RequestType": "Update",
    "ServiceToken": "arn:aws:lambda:ap-southeast-2:1234567:function:amplify-test9656353151-dev-1-authTriggerFn7FCFA449-xyz",
    "ResponseURL": "https://cloudformation-custom-resource-response-apsoutheast2.s3-ap-southeast-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aap-sou...00d761385cd62013820dc780bc07d07e021f77",
    "StackId": "arn:aws:cloudformation:ap-southeast-2:1234567:stack/amplify-test9656353151-dev-165513-AuthTriggerCustomLambdaStack-VZC2CVEFA8GA/00876ee0-93a7-11ec-b1d1-xyz",
    "RequestId": "82bf2ca0-061e-4895-ad70-xyz",
    "LogicalResourceId": "CustomAuthTriggerResource",
    "PhysicalResourceId": "2022/02/22/[$LATEST]f72b84eb...54d7ef4",
    "ResourceType": "Custom::CustomAuthTriggerResourceOutputs",
    ...
}
    1. According to this AWS documentation , manually construct a cURL SUCCESS request based on the observed information. Note that the cURL destination URL should be the the "ResponseURL" above. Since it is a custom resource, you cannot use CloudFormation CLI signal-resource to unblock it. The cURL request looks like:
    $ curl -H 'Content-Type: ''' -X PUT -d '{"Status": "SUCCESS","PhysicalResourceId": "2022/02/22/[$LATEST]f72b84eb...54d7ef4","StackId": "arn:aws:cloudformation:ap-southeast-2:1234567:stack/amplify-test9656353151-dev-165513-AuthTriggerCustomLambdaStack-VZC2CVEFA8GA/00876ee0-93a7-11ec-b1d1-xyz","RequestId": "82bf2ca0-061e-4895-ad70-xyz","LogicalResourceId": "CustomAuthTriggerResource"}' 'https://cloudformation-custom-resource-response-apsoutheast2.s3-ap-southeast-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aap-sou...00d761385cd62013820dc780bc07d07e021f77'
    1. Once you send the SUCCESS request out, CloudFormation stack will restore to UPDATE_ROLLBACK_COMPLETE state, and user can make further push.

    Then adjust the lambda timeout in both the Lambda and the CF managing the lambda to 300 seconds.