Because Personio's API won't tell me when employees are created or modified which creates scalability issues for us, I'm creating a cache where I can store the Personio with metadata including whether/when it has been modified.
I've already learned from @Charchit Kapoor I should be able to use $addFields with $function to populate a "lastModified" value, so now I'm trying to make this work within my C# application.
I've created the following Save method:
public Task<UpdateResult[]> Save(List<StoredDynamicRecord> personioRecords, CancellationToken cancellationToken)
{
IMongoCollection<StoredDynamicRecord> collection = GetCollection<StoredDynamicRecord>();
IEnumerable<Task<UpdateResult>> updateResultTasks
= personioRecords.Select(personioRecord =>
{
string id = (string)personioRecord.AttributesByName["id"].Value;
personioRecord.CacheId = string.IsNullOrEmpty(id)
? ObjectId.GenerateNewId()
.ToString()
: id;
personioRecord.SyncTimestamp = DateTime.UtcNow;
return collection.UpdateOneAsync(
filter: Builders<StoredDynamicRecord>.Filter.Eq(x => x.CacheId, personioRecord.CacheId),
update: new EmptyPipelineDefinition<StoredDynamicRecord>()
.AppendStage<StoredDynamicRecord, StoredDynamicRecord, StoredDynamicRecord>(
@$"{{ ""$replaceWith"":
{personioRecord.ToBsonDocument()}
}}"
)
.AppendStage<StoredDynamicRecord, StoredDynamicRecord, StoredDynamicRecord>(
@$"{{ ""$addFields"": {{ ""_lastModified"": {{ ""$function"": {{
""lang"": ""js"",
""args"": [
""$ROOT"",
{{
""key"": 1,
""data"": ""somedata""
}}
],
""body"": ""function(oldDoc, newDoc) {{
return (!oldDoc || JSON.stringify(oldDoc.AttributesByName) !== JSON.stringify(newDoc.AttributesByName))
? newDoc._syncTimestamp
: oldDoc._lastModified
}}""
}} }} }} }}"
),
options: new()
{
IsUpsert = true
},
cancellationToken
);
});
return Task.WhenAll(updateResultTasks);
}
However at least one thing is wrong with this since the value of "_lastModified" is always null, when I would expect it never should be. If there is no oldDoc, I would expect the value to be set to newDoc._syncTimestamp which should be the same as personioRecord.SyncTimestamp = DateTime.UtcNow.
If I switch the order of the AppendStages, the value is "ISODate("0001-01-01T00:00:00.000+0000")" instead of null, which is arguably better but still not what is expected or wanted.
Fwiw, this is the StoredDynamicRecord class:
public class StoredDynamicRecord : DynamicRecord, IStoredRecord
{
public const string SyncTimestampField = "_syncTimestamp";
public const string LastModifiedTimestampField = "_lastModified";
[BsonId]
public string CacheId { get; set; }
[BsonElement(SyncTimestampField)]
public DateTime SyncTimestamp { get; set; }
[BsonElement(LastModifiedTimestampField)]
public DateTime LastModified { get; set; }
public StoredDynamicRecord From(DynamicRecord dynamicRecord) =>
new()
{
Type = dynamicRecord.Type,
AttributesByName = dynamicRecord.AttributesByName
};
}
What am I missing or doing wrong?
Couldn't get the above approach to work.
This works, but in memory instead of the database:
public Task<StoredDynamicRecord[]> Save(List<StoredDynamicRecord> personioRecords, CancellationToken cancellationToken)
{
IMongoCollection<StoredDynamicRecord> employeeCollection = GetCollection<StoredDynamicRecord>();
List<Task<StoredDynamicRecord>> updateResultTasks = personioRecords.Select(personioRecord =>
{
Dictionary<string, DynamicRecordAttribute> attributesByName = personioRecord.AttributesByName;
string id = (string)attributesByName["id"].Value;
personioRecord.CacheId = string.IsNullOrEmpty(id)
? ObjectId.GenerateNewId()
.ToString()
: id;
DateTime utcNow = DateTime.UtcNow;
personioRecord.SyncTimestamp = utcNow;
StoredDynamicRecord oldRecord = employeeCollection
.Find(x => x.CacheId == personioRecord.CacheId)
.FirstOrDefault();
return employeeCollection.FindOneAndUpdateAsync(
filter: Builders<StoredDynamicRecord>.Filter.Eq(x => x.CacheId, personioRecord.CacheId),
update: Builders<StoredDynamicRecord>.Update
.Set(x => x.CacheId, string.IsNullOrEmpty(id)
? ObjectId.GenerateNewId()
.ToString()
: id
)
.Set(x => x.Type, personioRecord.Type)
.Set(x => x.AttributesByName, attributesByName)
.Set(x => x.SyncTimestamp, personioRecord.SyncTimestamp)
.Set(x => x.LastModified, (!personioRecord.Equals(oldRecord))
? utcNow
: oldRecord.LastModified
),
options: new FindOneAndUpdateOptions<StoredDynamicRecord>()
{
IsUpsert = true
},
cancellationToken
);
})
.ToList();
return Task.WhenAll(updateResultTasks);
}
Note, for this to work, it is important to define equality:
public override bool Equals(object that) =>
this == that
|| (
that != null
&& that is StoredDynamicRecord otherRecord
&& CacheId == otherRecord.CacheId
&& AttributesByName.Count == otherRecord.AttributesByName.Count
&& AttributesByName.Keys.ForAll(x =>
AttributesByName[x].ToString() == otherRecord.AttributesByName[x].ToString()
)
);