I am trying to create a DMS replication task using Terraform. The credentials for the endpoint are correct. When I start the replication task, it Fails but the cloudwatch logs aren't that helpful. This is what I see:
....
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]D: 51) name: 'varchar', SQLType: 12, create params: 'length', unsigned: '-1', iFixedPrecScale: '0', iColumnSize: '255', iMinimumScale '0', iMaximumScale: '0' (ar_odbc_conn.c:1371)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]D: Exit ODBC Provider supported data types (ar_odbc_conn.c:1391)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]I: Source endpoint 'Mysql' is using provider syntax 'MySQL' (provider_syntax_manager.c:894)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]I: Mysql endpoint initialization finished for the stream component (mysql_endpoint_imp.c:808)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]D: Entering 'mysql_endpoint_free_imp' (mysql_endpoint_imp.c:879)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SORTER ]I: Sorter inspect target state (sorter.c:519)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SORTER ]I: Target state checkpoint source id: 0 (sorter.c:524)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SORTER ]I: 'Stop reading when memory limit reached' is set to true (sorter.c:688)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [SOURCE_CAPTURE ]D: Leaving 'mysql_endpoint_free_imp' (mysql_endpoint_imp.c:896)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [TARGET_APPLY ]D: Entering 'odbc_endpoint_free_imp' (odbc_endpoint_imp.c:4045)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [TASK_MANAGER ]I: Subtask #0 ended (replicationtask_util.c:595)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [TASK_MANAGER ]I: Task - 222222222222222EEEE is in ERROR state, updating starting status to AR_NOT_APPLICABLE (repository.c:5483)
2024-05-24T07:42:20.000Z 2024-05-24T07:42:20 [TASK_MANAGER ]E: Task '222222222222222EEEEE' encountered a fatal error (repository.c:5580)
2024-05-24T07:42:29.000Z 2024-05-24T07:42:29 [TASK_MANAGER ]I: Task management thread terminated (replicationtask.c:4538)
2024-05-24T07:42:31.000Z 2024-05-24T07:42:31 [AT_GLOBAL ]I: Closing log file at Fri May 24 07:42:31 2024 (at_logger.c:2975)
but I am unable to comprehend it - what exactly is the issue?
Replication Task Mappings:
{
"rules": [
{
"rule-type": "selection",
"rule-id": "1",
"rule-name": "1",
"object-locator": {
"schema-name": "ti-lv",
"table-name": "application"
},
"rule-action": "include"
}
]
}
Replication Task Settings:
{
"TargetMetadata": {
"TargetSchema": "",
"SupportLobs": true,
"FullLobMode": false,
"LobChunkSize": 0,
"LimitedSizeLobMode": true,
"LobMaxSize": 512,
"InlineLobMaxSize": 0,
"LoadMaxFileSize": 0,
"ParallelLoadThreads": 0,
"ParallelLoadBufferSize": 0,
"BatchApplyEnabled": false,
"TaskRecoveryTableEnabled": false,
"ParallelLoadQueuesPerThread": 0,
"ParallelApplyThreads": 0,
"ParallelApplyBufferSize": 0,
"ParallelApplyQueuesPerThread": 0
},
"FullLoadSettings": {
"TargetTablePrepMode": "DROP_AND_CREATE",
"CreatePkAfterFullLoad": false,
"StopTaskCachedChangesApplied": false,
"StopTaskCachedChangesNotApplied": false,
"MaxFullLoadSubTasks": 2,
"TransactionConsistencyTimeout": 1200,
"CommitRate": 20000
},
"Logging": {
"EnableLogging": true,
"EnableLogContext": false,
"LogComponents": [
{
"Id": "TRANSFORMATION",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "SOURCE_UNLOAD",
"Severity": "LOGGER_SEVERITY_DEBUG"
},
{
"Id": "IO",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TARGET_LOAD",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "PERFORMANCE",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "SOURCE_CAPTURE",
"Severity": "LOGGER_SEVERITY_DEBUG"
},
{
"Id": "SORTER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "REST_SERVER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "VALIDATOR_EXT",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TARGET_APPLY",
"Severity": "LOGGER_SEVERITY_DEBUG"
},
{
"Id": "TASK_MANAGER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TABLES_MANAGER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "METADATA_MANAGER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "FILE_FACTORY",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "COMMON",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "ADDONS",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "DATA_STRUCTURE",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "COMMUNICATION",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "FILE_TRANSFER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
}
]
},
"ControlTablesSettings": {
"FullLoadExceptionTableEnabled": false,
"ControlSchema": "dms",
"HistoryTimeslotInMinutes": 5,
"HistoryTableEnabled": true,
"SuspendedTablesTableEnabled": true,
"StatusTableEnabled": true
},
"StreamBufferSettings": {
"StreamBufferCount": 3,
"StreamBufferSizeInMB": 8,
"CtrlStreamBufferSizeInMB": 5
},
"ChangeProcessingDdlHandlingPolicy": {
"HandleSourceTableDropped": true,
"HandleSourceTableTruncated": true,
"HandleSourceTableAltered": true
},
"ErrorBehavior": {
"EventErrorPolicy": "IGNORE",
"DataErrorPolicy": "LOG_ERROR",
"DataTruncationErrorPolicy": "LOG_ERROR",
"DataErrorEscalationPolicy": "SUSPEND_TABLE",
"DataErrorEscalationCount": 0,
"TableErrorPolicy": "SUSPEND_TABLE",
"TableErrorEscalationPolicy": "STOP_TASK",
"TableErrorEscalationCount": 0,
"RecoverableErrorCount": -1,
"RecoverableErrorInterval": 5,
"RecoverableErrorThrottling": true,
"RecoverableErrorThrottlingMax": 1800,
"RecoverableErrorStopRetryAfterThrottlingMax": true,
"ApplyErrorDeletePolicy": "IGNORE_RECORD",
"ApplyErrorInsertPolicy": "LOG_ERROR",
"ApplyErrorUpdatePolicy": "LOG_ERROR",
"ApplyErrorEscalationPolicy": "LOG_ERROR",
"ApplyErrorEscalationCount": 0,
"ApplyErrorFailOnTruncationDdl": false,
"FullLoadIgnoreConflicts": true,
"FailOnTransactionConsistencyBreached": false,
"FailOnNoTablesCaptured": false
},
"FailTaskWhenCleanTaskResourceFailed": false,
"TTSettings": {
"EnableTT": false,
"TTRecordSettings": null,
"TTS3Settings": null
},
"ChangeProcessingTuning": {
"BatchApplyPreserveTransaction": true,
"BatchApplyTimeoutMin": 1,
"BatchApplyTimeoutMax": 30,
"BatchApplyMemoryLimit": 500,
"BatchSplitSize": 0,
"MinTransactionSize": 1000,
"CommitTimeout": 1,
"MemoryLimitTotal": 1024,
"MemoryKeepTime": 60,
"StatementCacheSize": 50
},
"PostProcessingRules": null,
"CharacterSetSettings": null,
"LoopbackPreventionSettings": null,
"BeforeImageSettings": null
}
The problem was that the binary logs weren't activated on the MySQL database. I had to search "[E:" in the cloudwatch logs to filter out the relevant ones.