I have a number of EC2 instances in AWS that are backed up using the AWS Backup service. These instances all have a number of tags that are utilised for different purposes, from what backup schedule to apply, to automated startup/shutdown routines.
From the testing of the restore process and a couple of days trawling the internet it seems that these tags are not restored along with the instance.
In a DR scenario where a AZ is down and we need to restore these instances to a new AZ we would need the approprate tags in each of the restored instances.
Am I missing something with the backup/restore process?
If not, then I was looking into an automated process using a lambda function called from a cloudwatch event but I can't see an appropriate event for an instance restore (perhaps because it creates a new instance based on the backup?).
I'd need the tags and their values from the instance that was backed up created on the new restored instance, if there is an event for the restore I'd assume it would have the instanceid for both of these so I could get and set the tags, but if I can only use a create instance event I don't know how I can get the original tag values.
Anyone got a solution for this?
I managed to get a workable solution, with a C# lambda function triggered off a CloudWatch event when an instance is restored.
I've posted the solution below in case it helps someone else out, at least until they add tags as part of the restore process.
I followed this guide for creating the lambda function, but the main steps are below. In a cmd prompt:
dotnet new -i Amazon.Lambda.Templates
dotnet new lambda.EmptyFunction --name MySimpleFunction --profile default --region us-east-1
dotnet tool install -g Amazon.Lambda.Tools
Then open the solution and add a new class called AWS and add the below code. This does most of the work getting the tags from the old instance and adding to the new instance.
using System;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Amazon.EC2;
using Amazon.EC2.Model;
using Amazon.CloudTrail;
using Amazon.CloudTrail.Model;
using System.Threading.Tasks;
namespace RestoreEc2Tags
{
/// <summary>
/// Class for interacting with Amazon AWS services through the .Net SDK
/// </summary>
public class AWS
{
/// <summary>
/// Gets a list of all Ec2 Instances that do not have a name tag set, assumption being that these
/// will be the restored instances (missing all of the tags).
/// </summary>
/// <returns>List<tTag/></returns>
public async Task<List<Ec2Instance>> GetNewInstancesAsync()
{
AmazonEC2Client client = new AmazonEC2Client();
DescribeInstancesRequest ec2Request = new DescribeInstancesRequest();
DescribeInstancesResponse ec2Response = await client.DescribeInstancesAsync(ec2Request);
var instances = new List<Instance>();
foreach (var reservation in ec2Response.Reservations)
{
for (int i = 0; i < reservation.Instances.Count; i++)
{
instances.Add(reservation.Instances[i]);
}
}
var ec2Instances = new List<Ec2Instance>();
foreach (var instance in instances)
{
var ec2Instance = new Ec2Instance();
var tags = instance.Tags;
var nameTag = tags.Where(t => t.Key == "Name").FirstOrDefault();
bool isRestored = nameTag == null || nameTag.Value == null ? true : false;
ec2Instance.InstanceId = instance.InstanceId;
ec2Instance.State = instance.State.Name;
if (isRestored)
{
ec2Instances.Add(ec2Instance);
}
}
return ec2Instances;
}
/// <summary>
/// Gets all of the tags from the EC2 instance that the backup was from.
/// </summary>
/// <param name="originalInstanceId"></param>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns>List<tTag/></returns>
public async Task<List<Amazon.EC2.Model.Tag>> GetTagsAsync(string originalInstanceId, DateTime startTime, DateTime endTime)
{
var tagsToAdd = new List<Amazon.EC2.Model.Tag>();
var region = Amazon.RegionEndpoint.APSoutheast2;
var client = new AmazonCloudTrailClient(new AmazonCloudTrailConfig
{
Timeout = TimeSpan.FromSeconds(300), // Default value is 100 seconds
RegionEndpoint = region
});
LookupEventsRequest cloudTrailRequest = new LookupEventsRequest();
LookupAttribute eventName = new LookupAttribute { AttributeKey = "eventName", AttributeValue = "CreateTags" };
var lookupAttributes = new List<LookupAttribute>();
lookupAttributes.Add(eventName);
cloudTrailRequest.LookupAttributes = lookupAttributes;
cloudTrailRequest.StartTime = startTime;
cloudTrailRequest.EndTime = endTime;
var lookupEvents = await client.LookupEventsAsync(cloudTrailRequest);
var awsBackupCloudTrailEvent = lookupEvents.Events.Where(t => t.Username == "AWSBackup-AWSBackupDefaultServiceRole")
.Where(t => t.Resources
.Any(y => y.ResourceName.StartsWith("ami")))
.Where(t => t.CloudTrailEvent.Contains(originalInstanceId)).FirstOrDefault().CloudTrailEvent;
JObject o = JObject.Parse(awsBackupCloudTrailEvent);
JObject requestParameters = JObject.Parse(o["requestParameters"].ToString());
JObject tagSet = JObject.Parse(requestParameters["tagSet"].ToString());
var tagsFromCloudTrail = JsonConvert.DeserializeObject<JsonArray>(tagSet.ToString());
foreach (var item in tagsFromCloudTrail.Items)
{
var key = item["key"];
var value = "";
if (item.Count > 1)
{
value = item["value"];
}
var newTag = new Amazon.EC2.Model.Tag
{
Key = key,
Value = value
};
//aws: are reserved tags
if (!key.Contains("aws:"))
{
tagsToAdd.Add(newTag);
}
}
return tagsToAdd;
}
/// <summary>
/// Gets the InstanceId and Date of the newly restored EC2 instance from an AmiId. This Instance will need the tags restored to it.
/// </summary>
/// <param name="ami"></param>
/// <returns></returns>
public async Task<BackupDetails> GetBackupDetailsAsync(string ami)
{
var backupDetails = new BackupDetails();
var region = Amazon.RegionEndpoint.APSoutheast2;
var client = new AmazonCloudTrailClient(new AmazonCloudTrailConfig
{
Timeout = TimeSpan.FromSeconds(300), // Default value is 100 seconds
RegionEndpoint = region
});
LookupEventsRequest cloudTrailRequest = new LookupEventsRequest();
LookupAttribute eventName = new LookupAttribute { AttributeKey = "eventName", AttributeValue = "RestoreCompleted" };
var lookupAttributes = new List<LookupAttribute>();
lookupAttributes.Add(eventName);
cloudTrailRequest.LookupAttributes = lookupAttributes;
var lookupEvents = await client.LookupEventsAsync(cloudTrailRequest);
var awsBackupCloudTrailEvents = lookupEvents.Events.Where(t => t.CloudTrailEvent.Contains(ami)).FirstOrDefault();
if (awsBackupCloudTrailEvents != null)
{
var awsBackupCloudTrailEvent = awsBackupCloudTrailEvents.CloudTrailEvent;
JObject o = JObject.Parse(awsBackupCloudTrailEvent);
JObject serviceEventDetails = JObject.Parse(o["serviceEventDetails"].ToString());
var backupServiceEvent = JsonConvert.DeserializeObject<BackupServiceEvent>(serviceEventDetails.ToString());
var resourceArn = backupServiceEvent.resourceArn;
var backupCreationDateEpoch = backupServiceEvent.creationDate.seconds;
var backupCreationDate = UnixTimestampToDateTime(backupCreationDateEpoch);
var findString = "instance/";
int startIndex = resourceArn.IndexOf(findString, 0) + findString.Length;
int length = resourceArn.Length - startIndex;
backupDetails.InstanceId = resourceArn.Substring(startIndex, length);
backupDetails.CreationDate = awsBackupCloudTrailEvents.EventTime;
}
return backupDetails;
}
/// <summary>
/// Returns a DateTime type from a unix epoch timestamp.
/// </summary>
/// <param name="unixTime"></param>
/// <returns>DateTime</returns>
public static DateTime UnixTimestampToDateTime(double unixTime)
{
DateTime unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc);
long unixTimeStampInTicks = (long)(unixTime * TimeSpan.TicksPerSecond);
return new DateTime(unixStart.Ticks + unixTimeStampInTicks, System.DateTimeKind.Utc);
}
/// <summary>
/// Gets the AmiId that was used to restore an Instance. Required to find the tags from the original instance that were copied to the backup.
/// </summary>
/// <param name="newInstanceId"></param>
/// <returns>AmiId</returns>
public async Task<string> GetAmiIdFromNewInstanceAsync(string newInstanceId)
{
string amiId = "";
var region = Amazon.RegionEndpoint.APSoutheast2;
var client = new AmazonCloudTrailClient(new AmazonCloudTrailConfig
{
Timeout = TimeSpan.FromSeconds(300),
RegionEndpoint = region
});
LookupEventsRequest cloudTrailRequest = new LookupEventsRequest();
LookupAttribute eventName = new LookupAttribute { AttributeKey = "eventName", AttributeValue = "RunInstances" };
var lookupAttributes = new List<LookupAttribute>();
lookupAttributes.Add(eventName);
cloudTrailRequest.LookupAttributes = lookupAttributes;
var lookupEvents = await client.LookupEventsAsync(cloudTrailRequest);
var awsBackupCloudTrailEvents = lookupEvents.Events.Where(t => t.CloudTrailEvent.Contains(newInstanceId)).FirstOrDefault();
if (awsBackupCloudTrailEvents != null)
{
var awsBackupCloudTrailEvent = awsBackupCloudTrailEvents.CloudTrailEvent;
JObject o = JObject.Parse(awsBackupCloudTrailEvent);
JObject responseElements = JObject.Parse(o["responseElements"].ToString());
JObject instancesSet = JObject.Parse(responseElements["instancesSet"].ToString());
JObject instanceItems = JObject.Parse(instancesSet["items"].First.ToString());
var instance = JsonConvert.DeserializeObject<InstancesSet>(instanceItems.ToString());
amiId = instance.imageId;
}
return amiId;
}
/// <summary>
/// Restores all the tags to an EC2 instance that has been restored.
/// </summary>
/// <param name="instanceId"></param>
/// <param name="tags"></param>
/// <returns></returns>
public async Task<string> RestoreTagsAsync(string instanceId, List<Amazon.EC2.Model.Tag> tags)
{
var response = "";
try
{
AmazonEC2Client client = new AmazonEC2Client();
await client.CreateTagsAsync(new CreateTagsRequest
{
Resources = new List<string> {
instanceId
},
Tags = tags
});
response = "Tags restored successfully.";
}
catch (Exception ex)
{
response = String.Format("There has been an error: {0}:", ex.Message);
}
return response;
}
/// <summary>
/// Ties all the information from the CloudWatch logs together to get the tags associated with a restored instance
/// then calls the function to restore the tags
/// </summary>
/// <param name="newInstanceId"></param>
/// <returns></returns>
public async Task RestoreInstanceTagsAsync(string newInstanceId)
{
var ami = await GetAmiIdFromNewInstanceAsync(newInstanceId);
if (ami != "")
{
var backupDetails = await GetBackupDetailsAsync(ami);
if (backupDetails.InstanceId != null)
{
//Assuming daily backups, so we need to go back and get the last backup which could be up to 24 hours ago
var tags = await GetTagsAsync(backupDetails.InstanceId, backupDetails.CreationDate.AddHours(-24), backupDetails.CreationDate.AddHours(1));
await RestoreTagsAsync(newInstanceId, tags);
}
}
}
}
/// <summary>
/// Class for Deserializing Items collection from AWS json
/// </summary>
public class JsonArray
{
/// <summary>
/// Item collection from a json array retured from AWS
/// </summary>
public IEnumerable<IDictionary<string, string>> Items { get; set; }
}
/// <summary>
/// Class for Deserializing Backup AWS CloudTrail json logs
/// </summary>
public class BackupServiceEvent
{
/// <summary>
/// State of the backup from CloudTrail logs
/// </summary>
public string state { get; set; }
/// <summary>
/// restoreJobId of the backup from CloudTrail logs
/// </summary>
public string restoreJobId { get; set; }
/// <summary>
/// backupVaultName of the backup from CloudTrail logs
/// </summary>
public string backupVaultName { get; set; }
/// <summary>
/// backupVaultArn of the backup from CloudTrail logs
/// </summary>
public string backupVaultArn { get; set; }
/// <summary>
/// recoveryPointArn of the backup from CloudTrail logs
/// </summary>
public string recoveryPointArn { get; set; }
/// <summary>
/// resourceArn of the backup from CloudTrail logs
/// </summary>
public string resourceArn { get; set; }
/// <summary>
/// backupSizeInBytes of the backup from CloudTrail logs. This is used to retrieve the new InstanceId and backup CreationDate.
/// </summary>
public string backupSizeInBytes { get; set; }
/// <summary>
/// iamRoleArn of the backup from CloudTrail logs
/// </summary>
public string iamRoleArn { get; set; }
/// <summary>
/// resourceType of the backup from CloudTrail logs
/// </summary>
public string resourceType { get; set; }
/// <summary>
/// completionDate of the backup from CloudTrail logs
/// </summary>
public AwsDates completionDate { get; set; }
/// <summary>
/// creationDate of the backup from CloudTrail logs
/// </summary>
public AwsDates creationDate { get; set; }
}
/// <summary>
/// Class Deserializing Date structure from AWS CloudTrail json logs
/// </summary>
public class AwsDates
{
/// <summary>
/// Seconds from epoch from AWS dates
/// </summary>
public Int64 seconds { get; set; }
/// <summary>
/// Nano seconds, appended to seconds for precise epoch time
/// </summary>
public Int64 nanos { get; set; }
}
/// <summary>
/// Backup Details Class
/// </summary>
public class BackupDetails
{
/// <summary>
/// InstanceId that was backed up
/// </summary>
public string InstanceId { get; set; }
/// <summary>
/// Creation date of the backup
/// </summary>
public DateTime CreationDate { get; set; }
}
/// <summary>
/// Class for Deserializing Instance details from AWS CloudTrail json logs, or a newly created instance
/// </summary>
public class InstancesSet
{
/// <summary>
/// InstanceId of the newly created instance
/// </summary>
public string instanceId { get; set; }
/// <summary>
/// The AmiId used in the restore of an instance. Required to retrieve the origianal tags
/// </summary>
public string imageId { get; set; }
/// <summary>
/// instanceState of the newly created instance
/// </summary>
public Dictionary<string, string> instanceState { get; set; }
/// <summary>
/// privateDnsName of the newly created instance
/// </summary>
public string privateDnsName { get; set; }
/// <summary>
/// amiLaunchIndex of the newly created instance
/// </summary>
public int amiLaunchIndex { get; set; }
/// <summary>
/// instanceType of the newly created instance
/// </summary>
public string instanceType { get; set; }
/// <summary>
/// launchTime of the newly created instance
/// </summary>
public Int64 launchTime { get; set; }
/// <summary>
/// placement of the newly created instance
/// </summary>
public Dictionary<string, string> placement { get; set; }
/// <summary>
/// platform of the newly created instance
/// </summary>
public string platform { get; set; }
/// <summary>
/// monitoring of the newly created instance
/// </summary>
public Dictionary<string, string> monitoring { get; set; }
/// <summary>
/// subnetId of the newly created instance
/// </summary>
public string subnetId { get; set; }
/// <summary>
/// vpcId of the newly created instance
/// </summary>
public string vpcId { get; set; }
/// <summary>
/// privateIpAddress of the newly created instance
/// </summary>
public string privateIpAddress { get; set; }
/// <summary>
/// stateReason of the newly created instance
/// </summary>
public Dictionary<string, string> stateReason { get; set; }
/// <summary>
/// architecture of the newly created instance
/// </summary>
public string architecture { get; set; }
/// <summary>
/// rootDeviceType of the newly created instance
/// </summary>
public string rootDeviceType { get; set; }
/// <summary>
/// rootDeviceName of the newly created instance
/// </summary>
public string rootDeviceName { get; set; }
/// <summary>
/// blockDeviceMapping of the newly created instance
/// </summary>
public Dictionary<string, string> blockDeviceMapping { get; set; }
/// <summary>
/// virtualizationType of the newly created instance
/// </summary>
public string virtualizationType { get; set; }
/// <summary>
/// hypervisor of the newly created instance
/// </summary>
public string hypervisor { get; set; }
/// <summary>
/// clientToken of the newly created instance
/// </summary>
public string clientToken { get; set; }
}
public class Ec2Instance : IComparable<Ec2Instance>
{
public string Name { get; set; }
public string InstanceId { get; set; }
public string State { get; set; }
public string Environment { get; set; }
public string HostName { get; set; }
public string Type { get; set; }
public string StartStopSchedule { get; set; }
public string DisplaySchedule { get; set; }
public string PartnerInstanceId { get; set; }
public List<string> StartStopPermissions { get; set; }
public int CompareTo(Ec2Instance that)
{
return this.Name.CompareTo(that.Name);
}
}
}
Then replace the function handler in the function class with the below code:
/// <summary>
/// Gets all of the Ec2Instances that do not have a name tag and attempts to restore all of the tags
/// from the instance it was restored from. Triggered by RestoreCompleted CloudWatch log.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public async Task<string> FunctionHandler(ILambdaContext context)
{
var response = "Error?";
var aws = new AWS();
var newInstances = await aws.GetNewInstancesAsync();
foreach (var newInstance in newInstances)
{
await aws.RestoreInstanceTagsAsync(newInstance.InstanceId);
response = "Success";
}
return response;
}
After that you need to packge the solution as a zip file so you can upload to a lambda function:
dotnet lambda package -c Release -o ../RestoreEc2Tags.zip -f netcoreapp3.1
Then in your AWS console, create a new Lambda function, ensure you pick .net 3.1 framework, and upload the zip file.
Create a trigger for the function on CloudWatch Logs. You may have to set up a log if you do not have one already, and assign appropriate permissions to read/write to the logs for the IAM role used by the Lambda function.
Add a filter for the log trigger as below:
Filter pattern: { $.eventName = "RestoreCompleted" }
Hope this helps someone.