I'm encountering an issue while attempting to create a custom Oracle DB instance on AWS RDS. The creation process fails with the following error message indicating missing IAM permissions for various actions on different resources. Yet, the IAM with the correct set of permission (see policy AWSRDSCustomIamRolePolicyExample) is linked to the RDS Instance role (AWSRDSCustomInstanceRole-ap-southeast-1).
resource "aws_rds_custom_db_engine_version" "ora19-std-cev" {
database_installation_files_s3_bucket_name = aws_s3_bucket.cev_bucket.id
database_installation_files_s3_prefix = "stdcev1/"
engine = "custom-oracle-ee-cdb"
engine_version = "19.example.cdb_cev1"
kms_key_id = aws_kms_key.rds_custom_ora.arn
manifest = <<JSON
{
"databaseInstallationFileNames":["V982063-01.zip"],
"opatchFileNames":["p6880880_190000_Linux-x86-64.zip"],
"psuRuPatchFileNames":["p32126828_190000_Linux-x86-64.zip"],
"otherPatchFileNames":["p29213893_1910000DBRU_Generic.zip","p29782284_1910000DBRU_Generic.zip","p28730253_190000_Linux-x86-64.zip","p29374604_1910000DBRU_Linux-x86-64.zip","p28852325_190000_Linux-x86-64.zip","p29997937_190000_Linux-x86-64.zip","p31335037_190000_Linux-x86-64.zip","p31335142_190000_Generic.zip"]
}
JSON
}
data "aws_rds_orderable_db_instance" "custom-example-oracle" {
engine = "custom-oracle-ee-cdb" # CEV engine to be used
engine_version = "19.example.cdb_cev1" # CEV engine version to be used
license_model = "bring-your-own-license"
storage_type = "gp3"
preferred_instance_classes = ["db.r5.xlarge", "db.r5.2xlarge", "db.r5.4xlarge"]
depends_on = [
aws_rds_custom_db_engine_version.ora19-std-cev
]
}
resource "aws_db_instance_role_association" "db_inst_role_ora19inst-example" {
db_instance_identifier = aws_db_instance.ora19inst-example.identifier
feature_name = "ORA_EXAMPLE_S3_INTEGRATION"
role_arn = aws_iam_role.rds_custom_role.arn
}
resource "aws_db_instance" "ora19inst-example" {
allocated_storage = 500
auto_minor_version_upgrade = false # Custom for Oracle does not support minor version upgrades
custom_iam_instance_profile = "AWSRDSCustomInstanceProfile-ap-southeast-1" # Instance profile is required for Custom for Oracle. See: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/custom-setup-orcl.html#custom-setup-orcl.iam-vpc
backup_retention_period = 7
db_subnet_group_name = "db-oracle-private"
engine = data.aws_rds_orderable_db_instance.custom-example-oracle.engine
engine_version = data.aws_rds_orderable_db_instance.custom-example-oracle.engine_version
identifier = "ee-instance-test-example-1"
instance_class = data.aws_rds_orderable_db_instance.custom-example-oracle.instance_class
kms_key_id = aws_kms_key.rds_custom_ora.arn
license_model = data.aws_rds_orderable_db_instance.custom-example-oracle.license_model
multi_az = false # Custom for Oracle does not support multi-az
password = "avoid-plaintext-passwords"
username = "admin"
storage_encrypted = true
timeouts {
create = "1h"
delete = "3h"
update = "3h"
}
}
resource "aws_iam_instance_profile" "aws_rds_custom_instance_profile" {
name = "AWSRDSCustomInstanceProfile-ap-southeast-1"
role = aws_iam_role.rds_custom_role.name
}
data "aws_iam_policy_document" "assume_rds_role" {
statement {
effect = "Allow"
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
actions = ["sts:AssumeRole"]
}
}
resource "aws_iam_role" "rds_custom_role" {
name = "AWSRDSCustomInstanceRole-ap-southeast-1"
path = "/"
assume_role_policy = data.aws_iam_policy_document.assume_rds_role.json
}
resource "aws_iam_role_policy_attachment" "rds-policy-attachment" {
policy_arn = "arn:aws:iam::<removed>:policy/AWSRDSCustomIamRolePolicyExample"
role = aws_iam_role.rds_custom_role.name
}
{
"Statement": [
{
"Action": [
"ssm:DescribeAssociation",
"ssm:GetDeployablePatchSnapshotForInstance",
"ssm:GetDocument",
"ssm:DescribeDocument",
"ssm:GetManifest",
"ssm:GetParameter",
"ssm:GetParameters",
"ssm:ListAssociations",
"ssm:ListInstanceAssociations",
"ssm:PutInventory",
"ssm:PutComplianceItems",
"ssm:PutConfigurePackageResult",
"ssm:UpdateAssociationStatus",
"ssm:UpdateInstanceAssociationStatus",
"ssm:UpdateInstanceInformation",
"ssm:GetConnectionStatus",
"ssm:DescribeInstanceInformation",
"ssmmessages:CreateControlChannel",
"ssmmessages:CreateDataChannel",
"ssmmessages:OpenControlChannel",
"ssmmessages:OpenDataChannel"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"ec2messages:AcknowledgeMessage",
"ec2messages:DeleteMessage",
"ec2messages:FailMessage",
"ec2messages:GetEndpoint",
"ec2messages:GetMessages",
"ec2messages:SendReply"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"logs:PutRetentionPolicy",
"logs:PutLogEvents",
"logs:DescribeLogStreams",
"logs:DescribeLogGroups",
"logs:CreateLogStream",
"logs:CreateLogGroup"
],
"Effect": "Allow",
"Resource": "arn:aws:logs:ap-southeast-1:<removed>:log-group:rds-custom-instance*"
},
{
"Action": [
"s3:putObject",
"s3:getObject",
"s3:getObjectVersion"
],
"Effect": "Allow",
"Resource": "arn:aws:s3:::do-not-delete-rds-custom-*/*"
},
{
"Action": "cloudwatch:PutMetricData",
"Condition": {
"StringEquals": {
"cloudwatch:namespace": [
"RDSCustomForOracle/Agent"
]
}
},
"Effect": "Allow",
"Resource": "*"
},
{
"Action": "events:PutEvents",
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"secretsmanager:GetSecretValue",
"secretsmanager:DescribeSecret"
],
"Effect": "Allow",
"Resource": "arn:aws:secretsmanager:ap-southeast-1:<removed>:secret:do-not-delete-rds-custom-*"
},
{
"Action": "s3:ListBucketVersions",
"Effect": "Allow",
"Resource": "arn:aws:s3:::do-not-delete-rds-custom-*"
},
{
"Action": [
"ec2:CreateSnapshots"
],
"Condition": {
"StringEquals": {
"ec2:ResourceTag/AWSRDSCustom": "custom-oracle"
}
},
"Effect": "Allow",
"Resource": [
"arn:aws:ec2:*:*:instance/*",
"arn:aws:ec2:*:*:volume/*"
]
},
{
"Action": "ec2:CreateSnapshots",
"Effect": "Allow",
"Resource": "arn:aws:ec2:*::snapshot/*"
},
{
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Effect": "Allow",
"Resource": "arn:aws:kms:ap-southeast-1:<removed>:key/bfe45637-514d-4ec0-82cd-94cc3ebbdaca"
},
{
"Action": "ec2:CreateTags",
"Condition": {
"StringLike": {
"ec2:CreateAction": [
"CreateSnapshots"
]
}
},
"Effect": "Allow",
"Resource": "*"
}
],
"Version": "2012-10-17"
}
I get the following messages errors:
(i)Following 'terraform apply':
│ Error: waiting for RDS DB Instance (ee-instance-test-example-1) create: unexpected state 'incompatible-create', wanted target 'available, storage-optimization'. last error: %!s(<nil>)
│
│ with aws_db_instance.ora19inst-example,
│ on oracle-rds.tf line 280, in resource "aws_db_instance" "ora19inst-example":
│ 280: resource "aws_db_instance" "ora19inst-example" {
(ii) On AWS Console, under the tab "Log and Events":
You can't create the DB instance because of incompatible resources. The IAM instance profile role [AWSRDSCustomInstanceRole-ap-southeast-1] is missing the following permissions: EFFECT [Allow] on ACTION(S) [ssm:DescribeAssociation, ssm:DescribeDocument, ssm:GetConnectionStatus, ssm:GetDeployablePatchSnapshotForInstance, ssmmessages:OpenControlChannel, ssm:GetParameters, ssm:ListInstanceAssociations, ssm:PutConfigurePackageResult, ssmmessages:CreateControlChannel, ssm:GetParameter, ssm:UpdateAssociationStatus, ssm:GetManifest, ssmmessages:CreateDataChannel, ssm:PutInventory, ssm:UpdateInstanceInformation, ssm:DescribeInstanceInformation, ssmmessages:OpenDataChannel, ssm:GetDocument, ssm:ListAssociations, ssm:PutComplianceItems, ssm:UpdateInstanceAssociationStatus] for RESOURCE(S) [*], EFFECT [Allow] on ACTION(S) [ec2messages:DeleteMessage, ec2messages:FailMessage, ec2messages:GetEndpoint, ec2messages:AcknowledgeMessage, ec2messages:GetMessages, ec2messages:SendReply] for RESOURCE(S) [*], EFFECT [Allow] on ACTION(S) [logs:CreateLogStream, logs:DescribeLogStreams, logs:PutRetentionPolicy, logs:PutLogEvents, logs:CreateLogGroup] for RESOURCE(S) [arn:aws:logs:ap-southeast-1:<removed>:log-group:rds-custom-instance*], EFFECT [Allow] on ACTION(S) [s3:getObjectVersion, s3:getObject, s3:putObject] for RESOURCE(S) [arn:aws:s3:::do-not-delete-rds-custom-*/*], EFFECT [Allow] on ACTION(S) [cloudwatch:PutMetricData] for RESOURCE(S) [*] with CONDITION(S) [{Condition Key: [cloudwatch:namespace], Type: [StringEquals], Values: [RDSCustomForOracle/Agent]}], EFFECT [Allow] on ACTION(S) [events:PutEvents] for RESOURCE(S) [*], EFFECT [Allow] on ACTION(S) [secretsmanager:GetSecretValue, secretsmanager:DescribeSecret] for RESOURCE(S) [arn:aws:secretsmanager:ap-southeast-1:<removed>:secret:do-not-delete-rds-custom-*], EFFECT [Allow] on ACTION(S) [s3:ListBucketVersions] for RESOURCE(S) [arn:aws:s3:::do-not-delete-rds-custom-*], EFFECT [Allow] on ACTION(S) [ec2:CreateSnapshots] for RESOURCE(S) [arn:aws:ec2:*:*:instance/*, arn:aws:ec2:*:*:volume/*] with CONDITION(S) [{Condition Key: [ec2:ResourceTag/AWSRDSCustom], Type: [StringEquals], Values: [custom-oracle]}], EFFECT [Allow] on ACTION(S) [ec2:CreateSnapshots] for RESOURCE(S) [arn:aws:ec2:*::snapshot/*], EFFECT [Allow] on ACTION(S) [ec2:CreateTags] for RESOURCE(S) [*] with CONDITION(S) [{Condition Key: [ec2:CreateAction], Type: [StringLike], Values: [CreateSnapshots]}]
I listed the instance profiles in this account using AWS CLI (exact cmd: aws iam list-instance-profiles), the output shows there is an instance profile "AWSRDSCustomInstanceProfile-ap-southeast-1":
aws iam list-instance-profiles-for-role --role-name "AWSRDSCustomInstanceRole-ap-southeast-1"
{
"InstanceProfiles": [
{
"Path": "/",
"InstanceProfileName": "AWSRDSCustomInstanceProfile-ap-southeast-1",
"InstanceProfileId": "AIPA3FLD2IVP4D2NGYDUD",
"Arn": "arn:aws:iam::<removed>:instance-profile/AWSRDSCustomInstanceProfile-ap-southeast-1",
"CreateDate": "2024-02-07T14:25:28+00:00",
"Roles": [
{
"Path": "/",
"RoleName": "AWSRDSCustomInstanceRole-ap-southeast-1",
"RoleId": "AROA3FLD2IVPXSQN6HUG3",
"Arn": "arn:aws:iam::<removed>:role/AWSRDSCustomInstanceRole-ap-southeast-1",
"CreateDate": "2024-02-07T14:25:27+00:00",
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
}
]
},
]
}
Get Policies attached to the role "AWSRDSCustomInstanceRole-ap-southeast-1"
aws iam list-attached-role-policies --role-name "AWSRDSCustomInstanceRole-ap-southeast-1"
{
"AttachedPolicies": [
{
"PolicyName": "AWSRDSCustomIamRolePolicyExample",
"PolicyArn": "arn:aws:iam::<removed>:policy/AWSRDSCustomIamRolePolicyExample"
}
]
}
Yet, there aren't any inline policies listed with the role "AWSRDSCustomInstanceRole-ap-southeast-1"
aws iam list-role-policies --role-name "AWSRDSCustomInstanceRole-ap-southeast-1"
{
"PolicyNames": []
}
Amazon RDS Custom Team has answered to my support case and solved the issue, see below the reasons why it failed and the solution:
Your AWS Account has been allowlisted in the region ap-southeast-1 to create Custom Oracle instances with your existing SCP policies. Currently there is a limitation with role permission validation of customers with SCP policies containing condition keys.
In this use case where the AWS account was added to an allowlist for RDS Custom for Oracle, the account was allowlisted to bypass a service side pre-check. This pre-check verifies that the role of the user, defined by its IAM policy, has all of the necessary permissions to create the database and run subsequent automation (eg: database patching or point in time restore). If the customer has any SCPs in place, however, the pre-check always fails with a false negative.
This is regardless of whether those SCPs would have allowed or prohibited instance creation in this particular case. Adding the account to the allowlist is only to disable the pre-check for the customer to avoid the false negative, and does not disable or bypass the SCPs. This false negative behaviour from the pre-check is a defect and the service team is working on long term fix to avoid this. Should you want to onboard other AWS accounts or onboard into a different region, please reach-out to us, as we will need to allowlist you again.
The following is to address your previous queries and clarify this behavior.
Is this issue only affecting RDS Custom Oracle ? Yes, this issue only affects RDS Custom Oracle.
Do we know at which release this issue appeared ? This issue is not engine version based but account based. As mentioned, unfortunately, this is a service defect that requires manual intervention from the internal team to allowlist the Account.
Please kindly try creating the RDS Custom Oracle instance again using the same IAM instance profile role [AWSRDSCustomInstanceRole-ap-southeast-1] as previously.